?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob4fca3554ed64d84f13fa270de85b59a31e61db96
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 6/5/2024 Ver. 0.12.8 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
38
39 # URL Parsing
40 try:
41 from urllib.parse import urlparse, urlunparse;
42 except ImportError:
43 from urlparse import urlparse, urlunparse;
44
45 # Windows-specific setup
46 if os.name == 'nt':
47 if sys.version_info[0] == 2:
48 import codecs;
49 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
50 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
51 else:
52 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
53 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
54
55 hashlib_guaranteed = False;
56 # Environment setup
57 os.environ["PYTHONIOENCODING"] = "UTF-8";
58 os.environ["LC_CTYPE"] = "UTF-8";
59
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys.version_info[0] == 2:
62 try:
63 reload(sys);
64 sys.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
66 pass;
67
68 # CRC32 import
69 try:
70 from zlib import crc32;
71 except ImportError:
72 from binascii import crc32;
73
74 # Define FileNotFoundError for Python 2
75 try:
76 FileNotFoundError;
77 except NameError:
78 FileNotFoundError = IOError;
79
80 # RAR file support
81 rarfile_support = False;
82 try:
83 import rarfile;
84 rarfile_support = True;
85 except ImportError:
86 pass;
87
88 # 7z file support
89 py7zr_support = False;
90 try:
91 import py7zr;
92 py7zr_support = True;
93 except ImportError:
94 pass;
95
96 # TAR file checking
97 try:
98 from xtarfile import is_tarfile;
99 except ImportError:
100 try:
101 from safetar import is_tarfile;
102 except ImportError:
103 from tarfile import is_tarfile;
104
105 # TAR file module
106 try:
107 import xtarfile as tarfile;
108 except ImportError:
109 try:
110 import safetar as tarfile;
111 except ImportError:
112 import tarfile;
113
114 # Paramiko support
115 haveparamiko = False;
116 try:
117 import paramiko;
118 haveparamiko = True;
119 except ImportError:
120 pass;
121
122 # PySFTP support
123 havepysftp = False;
124 try:
125 import pysftp;
126 havepysftp = True;
127 except ImportError:
128 pass;
129
130 # Requests support
131 haverequests = False;
132 try:
133 import requests;
134 haverequests = True;
135 import urllib3;
136 logging.getLogger("urllib3").setLevel(logging.WARNING);
137 except ImportError:
138 pass;
139
140 # HTTP and URL parsing
141 try:
142 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
143 from urllib.parse import urlparse;
144 except ImportError:
145 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
146 from urlparse import urlparse;
147
148 # StringIO and BytesIO
149 try:
150 from io import StringIO, BytesIO;
151 except ImportError:
152 try:
153 from cStringIO import StringIO;
154 from cStringIO import StringIO as BytesIO;
155 except ImportError:
156 from StringIO import StringIO;
157 from StringIO import StringIO as BytesIO;
158
159 __use_pysftp__ = False;
160 if(not havepysftp):
161 __use_pysftp__ = False;
162 __file_format_name__ = "CatFile";
163 __program_name__ = "Py"+__file_format_name__;
164 __file_format_lower__ = __file_format_name__.lower();
165 __file_format_magic__ = __file_format_name__;
166 __file_format_len__ = len(__file_format_magic__);
167 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__ = "\x00";
169 __file_format_ver__ = "001";
170 __use_new_style__ = True;
171 __use_advanced_list__ = True;
172 __use_alt_inode__ = False;
173 __file_format_extension__ = ".cat";
174 '''
175 __file_format_name__ = "FastArchive";
176 __program_name__ = "Py" + __file_format_name__;
177 __file_format_lower__ = __file_format_name__.lower();
178 __file_format_magic__ = "FstArch";
179 __file_format_len__ = len(__file_format_magic__);
180 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
181 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
182 __file_format_ver__ = "001";
183 __use_new_style__ = True;
184 __use_advanced_list__ = False;
185 __use_alt_inode__ = False;
186 __file_format_extension__ = ".fast";
187 '''
188 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
189 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
190 __project__ = __program_name__;
191 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
192 __version_info__ = (0, 12, 8, "RC 1", 1);
193 __version_date_info__ = (2024, 6, 5, "RC 1", 1);
194 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
195 __revision__ = __version_info__[3];
196 __revision_id__ = "$Id$";
197 if(__version_info__[4] is not None):
198 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
199 if(__version_info__[4] is None):
200 __version_date_plusrc__ = __version_date__;
201 if(__version_info__[3] is not None):
202 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
203 if(__version_info__[3] is None):
204 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
205
206 PyBitness = platform.architecture();
207 if(PyBitness=="32bit" or PyBitness=="32"):
208 PyBitness = "32";
209 elif(PyBitness=="64bit" or PyBitness=="64"):
210 PyBitness = "64";
211 else:
212 PyBitness = "32";
213
214 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
215 if(platform.python_implementation()!=""):
216 py_implementation = platform.python_implementation();
217 if(platform.python_implementation()==""):
218 py_implementation = "Python";
219 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
220 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
221 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
222 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
223 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
224 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
225 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
226
227 compressionsupport = [];
228 try:
229 import gzip;
230 compressionsupport.append("gz");
231 compressionsupport.append("gzip");
232 except ImportError:
233 pass;
234 try:
235 import bz2;
236 compressionsupport.append("bz2");
237 compressionsupport.append("bzip2");
238 except ImportError:
239 pass;
240 try:
241 import lz4, lz4.frame;
242 compressionsupport.append("lz4");
243 except ImportError:
244 pass;
245 try:
246 import lzo;
247 compressionsupport.append("lzo");
248 compressionsupport.append("lzop");
249 except ImportError:
250 pass;
251 try:
252 import zstandard;
253 compressionsupport.append("zstd");
254 compressionsupport.append("zstandard");
255 except ImportError:
256 pass;
257 try:
258 import lzma;
259 compressionsupport.append("lzma");
260 compressionsupport.append("xz");
261 except ImportError:
262 try:
263 from backports import lzma;
264 compressionsupport.append("lzma");
265 compressionsupport.append("xz");
266 except ImportError:
267 pass;
268 try:
269 import brotli;
270 except ImportError:
271 pass;
272 try:
273 import blosc;
274 except ImportError:
275 pass;
276
277 compressionlist = ['auto'];
278 compressionlistalt = [];
279 outextlist = [];
280 outextlistwd = [];
281 if('gzip' in compressionsupport):
282 compressionlist.append('gzip');
283 compressionlistalt.append('gzip');
284 outextlist.append('gz');
285 outextlistwd.append('.gz');
286 if('bzip2' in compressionsupport):
287 compressionlist.append('bzip2');
288 compressionlistalt.append('bzip2');
289 outextlist.append('bz2');
290 outextlistwd.append('.bz2');
291 if('zstd' in compressionsupport):
292 compressionlist.append('zstd');
293 compressionlistalt.append('zstd');
294 outextlist.append('zst');
295 outextlistwd.append('.zst');
296 if('lz4' in compressionsupport):
297 compressionlist.append('lz4');
298 compressionlistalt.append('lz4');
299 outextlist.append('lz4');
300 outextlistwd.append('.lz4');
301 if('lzo' in compressionsupport):
302 compressionlist.append('lzo');
303 compressionlistalt.append('lzo');
304 outextlist.append('lzo');
305 outextlistwd.append('.lzo');
306 if('lzop' in compressionsupport):
307 compressionlist.append('lzop');
308 compressionlistalt.append('lzop');
309 outextlist.append('lzop');
310 outextlistwd.append('.lzop');
311 if('lzma' in compressionsupport):
312 compressionlist.append('lzma');
313 compressionlistalt.append('lzma');
314 outextlist.append('lzma');
315 outextlistwd.append('.lzma');
316 if('xz' in compressionsupport):
317 compressionlist.append('xz');
318 compressionlistalt.append('xz');
319 outextlist.append('xz');
320 outextlistwd.append('.xz');
321
322 tarfile_mimetype = "application/tar";
323 tarfile_tar_mimetype = tarfile_mimetype;
324 zipfile_mimetype = "application/zip";
325 zipfile_zip_mimetype = zipfile_mimetype;
326 rarfile_mimetype = "application/rar";
327 rarfile_rar_mimetype = rarfile_mimetype;
328 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
329 archivefile_cat_mimetype = archivefile_mimetype;
330 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
331 archivefile_gz_mimetype = archivefile_gzip_mimetype;
332 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
333 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
334 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
335 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
336 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
337 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
338 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
339 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
340 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
341 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
342
343 if __name__ == "__main__":
344 import subprocess;
345 curscrpath = os.path.dirname(sys.argv[0]);
346 if(curscrpath==""):
347 curscrpath = ".";
348 if(os.sep=="\\"):
349 curscrpath = curscrpath.replace(os.sep, "/");
350 curscrpath = curscrpath + "/";
351 scrfile = curscrpath + "catfile.py";
352 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
353 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
354 scrcmd.wait();
355
356 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
357 if(not dbgenable):
358 return True;
359 log_functions = {
360 "print": print,
361 "log": logging.info,
362 "warning": logging.warning,
363 "error": logging.error,
364 "critical": logging.critical,
365 "exception": logging.exception,
366 "logalt": lambda x: logging.log(dgblevel, x),
367 "debug": logging.debug
368 };
369 log_function = log_functions.get(outtype);
370 if(log_function):
371 log_function(dbgtxt);
372 return True;
373 return False;
374
375 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
376 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
377 return dbgtxt;
378
379 def RemoveWindowsPath(dpath):
380 if(dpath is None):
381 dpath = "";
382 if(os.sep!="/"):
383 dpath = dpath.replace(os.path.sep, "/");
384 dpath = dpath.rstrip("/");
385 if(dpath=="." or dpath==".."):
386 dpath = dpath + "/";
387 return dpath;
388
389 def NormalizeRelativePath(inpath):
390 inpath = RemoveWindowsPath(inpath);
391 if(os.path.isabs(inpath)):
392 outpath = inpath;
393 else:
394 if(inpath.startswith("./") or inpath.startswith("../")):
395 outpath = inpath;
396 else:
397 outpath = "./" + inpath;
398 return outpath;
399
400 def PrependPath(base_dir, child_path):
401 # Check if base_dir is None or empty, if so, return child_path as is
402 if not base_dir:
403 return child_path;
404 # Ensure base_dir ends with exactly one slash
405 if not base_dir.endswith('/'):
406 base_dir += '/';
407 # Check if child_path starts with ./ or ../ (indicating a relative path)
408 if child_path.startswith('./') or child_path.startswith('../'):
409 # For relative paths, we don't alter the child_path
410 return base_dir + child_path;
411 else:
412 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
413 return base_dir + child_path.lstrip('/');
414
415 def ListDir(dirpath, followlink=False, duplicates=False):
416 if isinstance(dirpath, (list, tuple, )):
417 dirpath = list(filter(None, dirpath));
418 elif isinstance(dirpath, str):
419 dirpath = list(filter(None, [dirpath]));
420 retlist = [];
421 fs_encoding = sys.getfilesystemencoding();
422 for mydirfile in dirpath:
423 if not os.path.exists(mydirfile):
424 return False;
425 mydirfile = NormalizeRelativePath(mydirfile);
426 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
427 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
428 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
429 for root, dirs, filenames in os.walk(mydirfile):
430 dpath = root;
431 dpath = RemoveWindowsPath(dpath);
432 if fs_encoding != 'utf-8':
433 dpath = dpath.encode(fs_encoding).decode('utf-8');
434 if dpath not in retlist and not duplicates:
435 retlist.append(dpath);
436 if duplicates:
437 retlist.append(dpath);
438 for file in filenames:
439 fpath = os.path.join(root, file);
440 fpath = RemoveWindowsPath(fpath);
441 if fs_encoding != 'utf-8':
442 fpath = fpath.encode(fs_encoding).decode('utf-8');
443 if fpath not in retlist and not duplicates:
444 retlist.append(fpath);
445 if duplicates:
446 retlist.append(fpath);
447 else:
448 path = RemoveWindowsPath(mydirfile);
449 if fs_encoding != 'utf-8':
450 path = path.encode(fs_encoding).decode('utf-8');
451 retlist.append(path);
452 return retlist;
453
454 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
455 if isinstance(dirpath, (list, tuple, )):
456 dirpath = list(filter(None, dirpath));
457 elif isinstance(dirpath, str):
458 dirpath = list(filter(None, [dirpath]));
459 retlist = [];
460 fs_encoding = sys.getfilesystemencoding();
461 for mydirfile in dirpath:
462 if not os.path.exists(mydirfile):
463 return False;
464 mydirfile = NormalizeRelativePath(mydirfile);
465 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
466 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
467 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
468 for root, dirs, filenames in os.walk(mydirfile):
469 # Sort dirs and filenames alphabetically in place
470 dirs.sort(key=lambda x: x.lower());
471 filenames.sort(key=lambda x: x.lower());
472 dpath = RemoveWindowsPath(root);
473 if fs_encoding != 'utf-8':
474 dpath = dpath.encode(fs_encoding).decode('utf-8');
475 if not duplicates and dpath not in retlist:
476 retlist.append(dpath);
477 elif duplicates:
478 retlist.append(dpath);
479 for file in filenames:
480 fpath = os.path.join(root, file);
481 fpath = RemoveWindowsPath(fpath);
482 if fs_encoding != 'utf-8':
483 fpath = fpath.encode(fs_encoding).decode('utf-8');
484 if not duplicates and fpath not in retlist:
485 retlist.append(fpath);
486 elif duplicates:
487 retlist.append(fpath);
488 else:
489 path = RemoveWindowsPath(mydirfile);
490 if fs_encoding != 'utf-8':
491 path = path.encode(fs_encoding).decode('utf-8');
492 retlist.append(path);
493 return retlist;
494
495 def create_alias_function(prefix, base_name, suffix, target_function):
496 # Define a new function that wraps the target function
497 def alias_function(*args, **kwargs):
498 return target_function(*args, **kwargs);
499 # Create the function name by combining the prefix, base name, and the suffix
500 function_name = "{}{}{}".format(prefix, base_name, suffix);
501 # Add the new function to the global namespace
502 globals()[function_name] = alias_function;
503
504 def create_alias_function_alt(prefix, base_name, suffix, target_function):
505 # Create the function name by combining the prefix, base name, and the suffix
506 # Use the format method for string formatting, compatible with Python 2 and 3
507 function_name = "{}{}{}".format(prefix, base_name, suffix)
508 # Add the new function (alias of the target_function) to the global namespace
509 # This line is compatible as-is with both Python 2 and 3
510 globals()[function_name] = target_function
511
512 def FormatSpecsListToDict(formatspecs=__file_format_list__):
513 if(isinstance(formatspecs, (list, tuple, ))):
514 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
515 elif(isinstance(formatspecs, (dict, ))):
516 return formatspecs;
517 else:
518 return __file_format_dict__;
519 return __file_format_dict__;
520
521 class ZlibFile:
522 def __init__(self, file_path, mode='rb', level=6):
523 self.file_path = file_path;
524 self.mode = mode;
525 self.level = level;
526 self._compressed_data = b'';
527 self._decompressed_data = b'';
528 self._position = 0;
529 if 'w' in mode:
530 self.file = open(file_path, mode);
531 elif 'r' in mode:
532 if os.path.exists(file_path):
533 self.file = open(file_path, mode);
534 self._load_file();
535 else:
536 raise FileNotFoundError(f"No such file: '{file_path}'");
537 else:
538 raise ValueError("Mode should be 'rb' or 'wb'");
539 def _load_file(self):
540 self.file.seek(0);
541 self._compressed_data = self.file.read();
542 self._decompressed_data = zlib.decompress(self._compressed_data);
543 def write(self, data):
544 compressed_data = zlib.compress(data, level=self.level);
545 self.file.write(compressed_data)
546 def read(self, size=-1):
547 if size == -1:
548 size = len(self._decompressed_data) - self._position;
549 data = self._decompressed_data[self._position:self._position + size];
550 self._position += size;
551 return data;
552 def seek(self, offset, whence=0):
553 if whence == 0: # absolute file positioning
554 self._position = offset;
555 elif whence == 1: # seek relative to the current position
556 self._position += offset;
557 elif whence == 2: # seek relative to the file's end
558 self._position = len(self._decompressed_data) + offset;
559 else:
560 raise ValueError("Invalid value for whence");
561 # Ensure the position is within bounds
562 self._position = max(0, min(self._position, len(self._decompressed_data)));
563 def tell(self):
564 return self._position;
565 def close(self):
566 self.file.close();
567 def __enter__(self):
568 return self;
569 def __exit__(self, exc_type, exc_value, traceback):
570 self.close();
571
572 class BloscFile:
573 def __init__(self, file_path, mode='rb', typesize=8):
574 self.file_path = file_path;
575 self.mode = mode;
576 self.typesize = typesize;
577 self._compressed_data = b'';
578 self._decompressed_data = b'';
579 self._position = 0;
580 if 'w' in mode:
581 self.file = open(file_path, mode);
582 elif 'r' in mode:
583 if os.path.exists(file_path):
584 self.file = open(file_path, mode);
585 self._load_file();
586 else:
587 raise FileNotFoundError(f"No such file: '{file_path}'");
588 else:
589 raise ValueError("Mode should be 'rb' or 'wb'");
590 def _load_file(self):
591 self.file.seek(0);
592 self._compressed_data = self.file.read();
593 self._decompressed_data = blosc.decompress(self._compressed_data);
594 def write(self, data):
595 compressed_data = blosc.compress(data, typesize=self.typesize);
596 self.file.write(compressed_data);
597 def read(self, size=-1):
598 if size == -1:
599 size = len(self._decompressed_data) - self._position;
600 data = self._decompressed_data[self._position:self._position + size];
601 self._position += size;
602 return data;
603 def seek(self, offset, whence=0):
604 if whence == 0: # absolute file positioning
605 self._position = offset;
606 elif whence == 1: # seek relative to the current position
607 self._position += offset;
608 elif whence == 2: # seek relative to the file's end
609 self._position = len(self._decompressed_data) + offset;
610 else:
611 raise ValueError("Invalid value for whence");
612 # Ensure the position is within bounds
613 self._position = max(0, min(self._position, len(self._decompressed_data)));
614 def tell(self):
615 return self._position;
616 def close(self):
617 self.file.close();
618 def __enter__(self):
619 return self;
620 def __exit__(self, exc_type, exc_value, traceback):
621 self.close();
622
623 class BrotliFile:
624 def __init__(self, file_path, mode='rb', quality=11):
625 self.file_path = file_path;
626 self.mode = mode;
627 self.quality = quality;
628 self._compressed_data = b'';
629 self._decompressed_data = b'';
630 self._position = 0;
631 if 'w' in mode:
632 self.file = open(file_path, mode);
633 elif 'r' in mode:
634 if os.path.exists(file_path):
635 self.file = open(file_path, mode);
636 self._load_file();
637 else:
638 raise FileNotFoundError(f"No such file: '{file_path}'");
639 else:
640 raise ValueError("Mode should be 'rb' or 'wb'");
641 def _load_file(self):
642 self.file.seek(0);
643 self._compressed_data = self.file.read();
644 self._decompressed_data = brotli.decompress(self._compressed_data);
645 def write(self, data):
646 compressed_data = brotli.compress(data, quality=self.quality);
647 self.file.write(compressed_data);
648 def read(self, size=-1):
649 if size == -1:
650 size = len(self._decompressed_data) - self._position;
651 data = self._decompressed_data[self._position:self._position + size];
652 self._position += size;
653 return data;
654 def seek(self, offset, whence=0):
655 if whence == 0: # absolute file positioning
656 self._position = offset;
657 elif whence == 1: # seek relative to the current position
658 self._position += offset;
659 elif whence == 2: # seek relative to the file's end
660 self._position = len(self._decompressed_data) + offset;
661 else:
662 raise ValueError("Invalid value for whence");
663 # Ensure the position is within bounds
664 self._position = max(0, min(self._position, len(self._decompressed_data)));
665 def tell(self):
666 return self._position;
667 def close(self):
668 self.file.close();
669 def __enter__(self):
670 return self;
671 def __exit__(self, exc_type, exc_value, traceback):
672 self.close();
673
674 def TarFileCheck(infile):
675 try:
676 if is_tarfile(infile):
677 return True;
678 else:
679 return False;
680 except TypeError:
681 try:
682 # Check if the input is a file object
683 if hasattr(infile, 'read'):
684 # Save the current file position
685 current_position = infile.tell();
686 # Attempt to open the file object as a tar file
687 tar = tarfile.open(fileobj=infile);
688 tar.close();
689 # Restore the file position
690 infile.seek(current_position);
691 return True;
692 else:
693 # Assume it's a filename
694 tar = tarfile.open(name=infile);
695 tar.close();
696 return True;
697 except tarfile.TarError:
698 return False;
699
700 # initial_value can be 0xFFFF or 0x0000
701 def crc16_ansi(msg, initial_value=0xFFFF):
702 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
703 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
704 crc = initial_value; # Initial value
705 for b in msg:
706 crc ^= b << 8; # XOR byte into CRC top byte
707 for _ in range(8): # Process each bit
708 if crc & 0x8000: # If the top bit is set
709 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
710 else:
711 crc = crc << 1; # Just shift left
712 crc &= 0xFFFF; # Ensure CRC remains 16-bit
713 return crc;
714
715 # initial_value can be 0xFFFF or 0x0000
716 def crc16_ibm(msg, initial_value=0xFFFF):
717 return crc16_ansi(msg, initial_value);
718
719 # initial_value is 0xFFFF
720 def crc16(msg):
721 return crc16_ansi(msg, 0xFFFF);
722
723 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
724 def crc16_ccitt(msg, initial_value=0xFFFF):
725 # CRC-16-CCITT polynomial
726 poly = 0x1021; # Polynomial for CRC-16-CCITT
727 # Use the specified initial value
728 crc = initial_value;
729 for b in msg:
730 crc ^= b << 8; # XOR byte into CRC top byte
731 for _ in range(8): # Process each bit
732 if crc & 0x8000: # If the top bit is set
733 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
734 else:
735 crc = crc << 1; # Just shift left
736 crc &= 0xFFFF; # Ensure CRC remains 16-bit
737 return crc;
738
739 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
740 def crc64_ecma(msg, initial_value=0x0000000000000000):
741 # CRC-64-ECMA polynomial and initial value
742 poly = 0x42F0E1EBA9EA3693;
743 crc = initial_value; # Initial value for CRC-64-ECMA
744 for b in msg:
745 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
746 for _ in range(8): # Process each bit
747 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
748 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
749 else:
750 crc <<= 1; # Just shift left if the MSB is 0
751 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
752 return crc;
753
754 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
755 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
756 # CRC-64-ISO polynomial and initial value
757 poly = 0x000000000000001B;
758 crc = initial_value; # Common initial value for CRC-64-ISO
759 for b in msg:
760 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
761 for _ in range(8): # Process each bit
762 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
763 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
764 else:
765 crc <<= 1; # Just shift left if the MSB is 0
766 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
767 return crc;
768
769 def GetDataFromArray(data, path, default=None):
770 element = data;
771 try:
772 for key in path:
773 element = element[key];
774 return element;
775 except (KeyError, TypeError, IndexError):
776 return default;
777
778 def GetDataFromArrayAlt(structure, path, default=None):
779 element = structure;
780 for key in path:
781 if isinstance(element, dict) and key in element:
782 element = element[key];
783 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
784 element = element[key];
785 else:
786 return default;
787 return element;
788
789 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
790 formatspecs = FormatSpecsListToDict(formatspecs);
791 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
792 if encodedata:
793 fileheader = fileheader.encode('UTF-8');
794 checksum_methods = {
795 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
796 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
797 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
798 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
799 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
800 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
801 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
802 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
803 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
804 };
805 if checksumtype in checksum_methods:
806 return checksum_methods[checksumtype](fileheader);
807 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
808 checksumoutstr = hashlib.new(checksumtype);
809 checksumoutstr.update(fileheader);
810 return checksumoutstr.hexdigest().lower();
811 return format(0, 'x').lower();
812
813 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
814 formatspecs = FormatSpecsListToDict(formatspecs);
815 if encodedata:
816 instr = instr.encode('UTF-8');
817 checksum_methods = {
818 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
819 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
820 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
821 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
822 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
823 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
824 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
825 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
826 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
827 };
828 if checksumtype in checksum_methods:
829 return checksum_methods[checksumtype](instr);
830 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
831 checksumoutstr = hashlib.new(checksumtype);
832 checksumoutstr.update(instr);
833 return checksumoutstr.hexdigest().lower();
834 return format(0, 'x').lower();
835
836 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
837 formatspecs = FormatSpecsListToDict(formatspecs);
838 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
839 return inchecksum.lower() == catfileheadercshex;
840
841 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
842 formatspecs = FormatSpecsListToDict(formatspecs);
843 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
844 return inchecksum.lower() == catinfilecshex;
845
846 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
847 curbyte = b"";
848 curfullbyte = b"";
849 nullbyte = delimiter.encode("UTF-8");
850 while(True):
851 curbyte = fp.read(1);
852 if(curbyte==nullbyte or not curbyte):
853 break;
854 curfullbyte = curfullbyte + curbyte;
855 return curfullbyte.decode('UTF-8');
856
857 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
858 return ReadTillNullByteOld(fp, delimiter);
859
860 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
861 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
862 buffer = bytearray();
863 total_read = 0;
864 delimiter_length = len(delimiter);
865 while True:
866 chunk = fp.read(chunk_size)
867 if not chunk:
868 # End of file reached without finding the delimiter
869 break;
870 buffer.extend(chunk);
871 total_read += len(chunk);
872 if delimiter in buffer:
873 # Delimiter found, calculate where to reset the file pointer
874 index = buffer.find(delimiter);
875 # Calculate how many extra bytes were read after the delimiter
876 extra_bytes_read = len(buffer) - (index + delimiter_length);
877 # Move the file pointer back to just after the delimiter
878 fp.seek(-extra_bytes_read, 1);
879 buffer = buffer[:index];
880 break;
881 if total_read >= max_read:
882 # Stop reading if max limit is reached to prevent excessive memory usage
883 raise MemoryError("Maximum read limit reached without finding the delimiter.");
884 # Check for incomplete UTF-8 sequences at the end of the buffer
885 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
886 # This suggests that the last byte might be the start of a multi-byte character
887 # Try to read one more byte to complete the character
888 extra_byte = fp.read(1);
889 if extra_byte:
890 buffer.extend(extra_byte);
891 else:
892 # No more data available
893 break;
894 try:
895 return buffer.decode('UTF-8', errors='replace');
896 except UnicodeDecodeError:
897 return buffer.decode('UTF-8', errors='replace');
898
899 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
900 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
901
902 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
903 curfullbyte = bytearray();
904 nullbyte = delimiter.encode("UTF-8");
905 total_read = 0; # Track the total number of bytes read
906 while True:
907 curbyte = fp.read(1);
908 if curbyte == nullbyte or not curbyte:
909 break;
910 curfullbyte.extend(curbyte);
911 total_read += 1;
912 if total_read >= max_read:
913 raise MemoryError("Maximum read limit reached without finding the delimiter.");
914 # Decode the full byte array to string once out of the loop
915 try:
916 return curfullbyte.decode('UTF-8');
917 except UnicodeDecodeError:
918 # Handle potential partial UTF-8 characters
919 for i in range(1, 4):
920 try:
921 return curfullbyte[:-i].decode('UTF-8');
922 except UnicodeDecodeError:
923 continue;
924 raise; # Re-raise if decoding fails even after trimming
925
926 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
927 return ReadTillNullByte(fp, delimiter, max_read);
928
929 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
930 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
931 buffer = bytearray();
932 total_read = 0;
933 delimiter_length = len(delimiter);
934 results = [];
935 while len(results) < num_delimiters:
936 chunk = fp.read(chunk_size);
937 if not chunk:
938 # End of file reached; decode whatever is collected if it's the last needed part
939 if len(buffer) > 0:
940 results.append(buffer.decode('UTF-8', errors='replace'));
941 break;
942 buffer.extend(chunk)
943 total_read += len(chunk);
944 # Check if we have found the delimiter
945 while delimiter in buffer:
946 index = buffer.find(delimiter);
947 # Decode the section before the delimiter
948 results.append(buffer[:index].decode('UTF-8', errors='replace'));
949 # Remove the processed part from the buffer
950 buffer = buffer[index + delimiter_length:];
951 if len(results) == num_delimiters:
952 # If reached the required number of delimiters, adjust the file pointer and stop
953 fp.seek(-len(buffer), 1);
954 return results;
955 if total_read >= max_read:
956 # Stop reading if max limit is reached to prevent excessive memory usage
957 raise MemoryError("Maximum read limit reached without finding the delimiter.");
958 # Check for incomplete UTF-8 sequences at the end of the buffer
959 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
960 # This suggests that the last byte might be the start of a multi-byte character
961 # Try to read one more byte to complete the character
962 extra_byte = fp.read(1);
963 if extra_byte:
964 buffer.extend(extra_byte);
965 else:
966 # No more data available
967 break;
968 # Process remaining buffer if less than the required number of delimiters were found
969 if len(buffer) > 0 and len(results) < num_delimiters:
970 results.append(buffer.decode('UTF-8', errors='replace'));
971 return results;
972
973 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
974 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
975
976 def SeekToEndOfFile(fp):
977 lasttell = 0;
978 while(True):
979 fp.seek(1, 1);
980 if(lasttell==fp.tell()):
981 break;
982 lasttell = fp.tell();
983 return True;
984
985 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
986 rocount = 0;
987 roend = int(rounds);
988 HeaderOut = [];
989 while(rocount<roend):
990 HeaderOut.append(ReadTillNullByte(fp, delimiter));
991 rocount = rocount + 1;
992 return HeaderOut;
993
994 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
995 headerpresize = ReadTillNullByte(fp, delimiter);
996 headersize = int(headerpresize, 16);
997 if(headersize<=0):
998 return [];
999 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
1000 fp.seek(1, 1);
1001 rocount = 0;
1002 roend = int(len(headercontent));
1003 HeaderOut = [headerpresize];
1004 while(rocount<roend):
1005 HeaderOut.append(headercontent[rocount]);
1006 rocount = rocount + 1;
1007 return HeaderOut;
1008
1009 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
1010 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
1011 headersize = int(preheaderdata[0], 16);
1012 headernumfields = int(preheaderdata[1], 16);
1013 if(headersize<=0 or headernumfields<=0):
1014 return [];
1015 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
1016 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
1017 HeaderOut = preheaderdata + headerdata;
1018 return HeaderOut;
1019
1020 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1021 formatspecs = FormatSpecsListToDict(formatspecs);
1022 delimiter = formatspecs['format_delimiter'];
1023 fheaderstart = fp.tell();
1024 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1025 if(len(HeaderOut)==0):
1026 return False;
1027 if(re.findall(r"^[.|/]", HeaderOut[3])):
1028 fname = HeaderOut[3];
1029 else:
1030 fname = "./"+HeaderOut[3];
1031 fcs = HeaderOut[-2].lower();
1032 fccs = HeaderOut[-1].lower();
1033 fsize = int(HeaderOut[5], 16);
1034 fcompression = HeaderOut[12];
1035 fcsize = int(HeaderOut[13], 16);
1036 fseeknextfile = HeaderOut[25];
1037 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1038 if(fcs!=newfcs and not skipchecksum):
1039 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1040 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1041 return False;
1042 fhend = fp.tell() - 1;
1043 fcontentstart = fp.tell();
1044 fcontents = BytesIO();
1045 if(fsize>0 and not listonly):
1046 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1047 fcontents.write(fp.read(fsize));
1048 else:
1049 fcontents.write(fp.read(fcsize));
1050 elif(fsize>0 and listonly):
1051 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1052 fp.seek(fsize, 1);
1053 else:
1054 fp.seek(fcsize, 1);
1055 fcontents.seek(0, 0);
1056 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1057 if(fccs!=newfccs and not skipchecksum and not listonly):
1058 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1059 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1060 return False;
1061 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1062 pass;
1063 else:
1064 fcontents.seek(0, 0);
1065 if(uncompress):
1066 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1067 fcontentend = fp.tell();
1068 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1069 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1070 if(abs(fseeknextasnum)==0):
1071 pass;
1072 fp.seek(fseeknextasnum, 1);
1073 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1074 fseeknextasnum = int(fseeknextfile);
1075 if(abs(fseeknextasnum)==0):
1076 pass;
1077 fp.seek(fseeknextasnum, 1);
1078 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1079 fseeknextasnum = int(fseeknextfile);
1080 if(abs(fseeknextasnum)==0):
1081 pass;
1082 fp.seek(fseeknextasnum, 0);
1083 else:
1084 return False;
1085 HeaderOut.append(fcontents);
1086 return HeaderOut;
1087
1088 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1089 formatspecs = FormatSpecsListToDict(formatspecs);
1090 delimiter = formatspecs['format_delimiter'];
1091 fheaderstart = fp.tell();
1092 if(formatspecs['new_style']):
1093 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1094 else:
1095 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1096 if(len(HeaderOut)==0):
1097 return False;
1098 fheadsize = int(HeaderOut[0], 16);
1099 fnumfields = int(HeaderOut[1], 16);
1100 ftype = int(HeaderOut[2], 16);
1101 if(re.findall(r"^[.|/]", HeaderOut[3])):
1102 fname = HeaderOut[3];
1103 else:
1104 fname = "./"+HeaderOut[3];
1105 fbasedir = os.path.dirname(fname);
1106 flinkname = HeaderOut[4];
1107 fsize = int(HeaderOut[5], 16);
1108 fatime = int(HeaderOut[6], 16);
1109 fmtime = int(HeaderOut[7], 16);
1110 fctime = int(HeaderOut[8], 16);
1111 fbtime = int(HeaderOut[9], 16);
1112 fmode = int(HeaderOut[10], 16);
1113 fchmode = stat.S_IMODE(fmode);
1114 ftypemod = stat.S_IFMT(fmode);
1115 fwinattributes = int(HeaderOut[11], 16);
1116 fcompression = HeaderOut[12];
1117 fcsize = int(HeaderOut[13], 16);
1118 fuid = int(HeaderOut[14], 16);
1119 funame = HeaderOut[15];
1120 fgid = int(HeaderOut[16], 16);
1121 fgname = HeaderOut[17];
1122 fid = int(HeaderOut[18], 16);
1123 finode = int(HeaderOut[19], 16);
1124 flinkcount = int(HeaderOut[20], 16);
1125 fdev_minor = int(HeaderOut[21], 16);
1126 fdev_major = int(HeaderOut[22], 16);
1127 frdev_minor = int(HeaderOut[23], 16);
1128 frdev_major = int(HeaderOut[24], 16);
1129 fseeknextfile = HeaderOut[25];
1130 fextrasize = int(HeaderOut[26], 16);
1131 fextrafields = int(HeaderOut[27], 16);
1132 extrafieldslist = [];
1133 extrastart = 28
1134 extraend = extrastart + fextrafields;
1135 extrafieldslist = [];
1136 if(extrastart<extraend):
1137 extrafieldslist.append(HeaderOut[extrastart]);
1138 extrastart = extrastart + 1;
1139 fcs = HeaderOut[-2].lower();
1140 fccs = HeaderOut[-1].lower();
1141 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1142 if(fcs!=newfcs and not skipchecksum):
1143 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1144 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1145 return False;
1146 fhend = fp.tell() - 1;
1147 fcontentstart = fp.tell();
1148 fcontents = BytesIO();
1149 pyhascontents = False;
1150 if(fsize>0 and not listonly):
1151 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1152 fcontents.write(fp.read(fsize));
1153 else:
1154 fcontents.write(fp.read(fcsize));
1155 pyhascontents = True;
1156 elif(fsize>0 and listonly):
1157 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1158 fp.seek(fsize, 1);
1159 else:
1160 fp.seek(fcsize, 1);
1161 pyhascontents = False;
1162 fcontents.seek(0, 0);
1163 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1164 if(fccs!=newfccs and not skipchecksum and not listonly):
1165 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1166 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1167 return False;
1168 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1169 pass;
1170 else:
1171 fcontents.seek(0, 0);
1172 if(uncompress):
1173 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1174 fcontents.seek(0, 0);
1175 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1176 fcontentend = fp.tell() - 1;
1177 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1178 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1179 if(abs(fseeknextasnum)==0):
1180 pass;
1181 fp.seek(fseeknextasnum, 1);
1182 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1183 fseeknextasnum = int(fseeknextfile);
1184 if(abs(fseeknextasnum)==0):
1185 pass;
1186 fp.seek(fseeknextasnum, 1);
1187 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1188 fseeknextasnum = int(fseeknextfile);
1189 if(abs(fseeknextasnum)==0):
1190 pass;
1191 fp.seek(fseeknextasnum, 0);
1192 else:
1193 return False;
1194 fcontents.seek(0, 0);
1195 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
1196 return catlist;
1197
1198 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1199 formatspecs = FormatSpecsListToDict(formatspecs);
1200 delimiter = formatspecs['format_delimiter'];
1201 fheaderstart = fp.tell();
1202 if(formatspecs['new_style']):
1203 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1204 else:
1205 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1206 if(len(HeaderOut)==0):
1207 return False;
1208 fheadsize = int(HeaderOut[0], 16);
1209 fnumfields = int(HeaderOut[1], 16);
1210 ftype = int(HeaderOut[2], 16);
1211 if(re.findall(r"^[.|/]", HeaderOut[3])):
1212 fname = HeaderOut[3];
1213 else:
1214 fname = "./"+HeaderOut[3];
1215 fbasedir = os.path.dirname(fname);
1216 flinkname = HeaderOut[4];
1217 fsize = int(HeaderOut[5], 16);
1218 fatime = int(HeaderOut[6], 16);
1219 fmtime = int(HeaderOut[7], 16);
1220 fctime = int(HeaderOut[8], 16);
1221 fbtime = int(HeaderOut[9], 16);
1222 fmode = int(HeaderOut[10], 16);
1223 fchmode = stat.S_IMODE(fmode);
1224 ftypemod = stat.S_IFMT(fmode);
1225 fwinattributes = int(HeaderOut[11], 16);
1226 fcompression = HeaderOut[12];
1227 fcsize = int(HeaderOut[13], 16);
1228 fuid = int(HeaderOut[14], 16);
1229 funame = HeaderOut[15];
1230 fgid = int(HeaderOut[16], 16);
1231 fgname = HeaderOut[17];
1232 fid = int(HeaderOut[18], 16);
1233 finode = int(HeaderOut[19], 16);
1234 flinkcount = int(HeaderOut[20], 16);
1235 fdev_minor = int(HeaderOut[21], 16);
1236 fdev_major = int(HeaderOut[22], 16);
1237 frdev_minor = int(HeaderOut[23], 16);
1238 frdev_major = int(HeaderOut[24], 16);
1239 fseeknextfile = HeaderOut[25];
1240 fextrasize = int(HeaderOut[26], 16);
1241 fextrafields = int(HeaderOut[27], 16);
1242 extrafieldslist = [];
1243 extrastart = 28;
1244 extraend = extrastart + fextrafields;
1245 extrafieldslist = [];
1246 if(extrastart<extraend):
1247 extrafieldslist.append(HeaderOut[extrastart]);
1248 extrastart = extrastart + 1;
1249 fheaderchecksumtype = HeaderOut[extrastart].lower();
1250 fcontentchecksumtype = HeaderOut[extrastart + 1].lower();
1251 fcs = HeaderOut[-2].lower();
1252 fccs = HeaderOut[-1].lower();
1253 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1254 if(fcs!=newfcs and not skipchecksum):
1255 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1256 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1257 return False;
1258 fhend = fp.tell() - 1;
1259 fcontentstart = fp.tell();
1260 fcontents = BytesIO();
1261 pyhascontents = False;
1262 if(fsize>0 and not listonly):
1263 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1264 fcontents.write(fp.read(fsize));
1265 else:
1266 fcontents.write(fp.read(fcsize));
1267 pyhascontents = True;
1268 elif(fsize>0 and listonly):
1269 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1270 fp.seek(fsize, 1);
1271 else:
1272 fp.seek(fcsize, 1);
1273 pyhascontents = False;
1274 fcontents.seek(0, 0);
1275 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1276 if(fccs!=newfccs and not skipchecksum and not listonly):
1277 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1278 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1279 return False;
1280 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1281 pass;
1282 else:
1283 fcontents.seek(0, 0);
1284 if(uncompress):
1285 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1286 fcontents.seek(0, 0);
1287 fcontentend = fp.tell() - 1;
1288 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1289 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1290 if(abs(fseeknextasnum)==0):
1291 pass;
1292 fp.seek(fseeknextasnum, 1);
1293 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1294 fseeknextasnum = int(fseeknextfile);
1295 if(abs(fseeknextasnum)==0):
1296 pass;
1297 fp.seek(fseeknextasnum, 1);
1298 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1299 fseeknextasnum = int(fseeknextfile);
1300 if(abs(fseeknextasnum)==0):
1301 pass;
1302 fp.seek(fseeknextasnum, 0);
1303 else:
1304 return False;
1305 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1306 return catlist;
1307
1308 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1309 formatspecs = FormatSpecsListToDict(formatspecs);
1310 delimiter = formatspecs['format_delimiter'];
1311 curloc = fp.tell();
1312 if(curloc>0):
1313 fp.seek(0, 0);
1314 catheader = ReadFileHeaderData(fp, 4, delimiter);
1315 if(curloc>0):
1316 fp.seek(curloc, 0);
1317 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1318 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1319 if(not headercheck and not skipchecksum):
1320 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1321 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1322 return False;
1323 fnumfiles = int(catheader[1], 16);
1324 countnum = 0;
1325 flist = [];
1326 while(countnum < fnumfiles):
1327 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1328 if(len(HeaderOut)==0):
1329 break;
1330 flist.append(HeaderOut);
1331 countnum = countnum + 1;
1332 return flist;
1333
1334 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1335 formatspecs = FormatSpecsListToDict(formatspecs);
1336 delimiter = formatspecs['format_delimiter'];
1337 curloc = fp.tell();
1338 if(curloc>0):
1339 fp.seek(0, 0);
1340 catheader = ReadFileHeaderData(fp, 4, delimiter);
1341 if(curloc>0):
1342 fp.seek(curloc, 0);
1343 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1344 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1345 if(not headercheck and not skipchecksum):
1346 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1347 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1348 return False;
1349 catstring = catheader[0];
1350 catversion = re.findall(r"([\d]+)", catstring);
1351 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1352 fprenumfiles = catheader[1];
1353 fnumfiles = int(fprenumfiles, 16);
1354 fprechecksumtype = catheader[2];
1355 fprechecksum = catheader[3];
1356 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1357 if(seekstart<0 and seekstart>fnumfiles):
1358 seekstart = 0;
1359 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1360 seekend = fnumfiles;
1361 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1362 seekend = fnumfiles - abs(seekend);
1363 if(seekstart>0):
1364 il = 0;
1365 while(il < seekstart):
1366 prefhstart = fp.tell();
1367 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1368 if(len(preheaderdata)==0):
1369 break;
1370 prefsize = int(preheaderdata[5], 16);
1371 prefseeknextfile = preheaderdata[25];
1372 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1373 prefcs = preheaderdata[-2];
1374 if(prefcs!=prenewfcs and not skipchecksum):
1375 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1376 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1377 return False;
1378 valid_archive = False;
1379 invalid_archive = True;
1380 prefhend = fp.tell() - 1;
1381 prefcontentstart = fp.tell();
1382 prefcontents = BytesIO();
1383 pyhascontents = False;
1384 if(prefsize>0):
1385 prefcontents.write(fp.read(prefsize));
1386 prefcontents.seek(0, 0);
1387 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1388 prefccs = preheaderdata[-1];
1389 pyhascontents = True;
1390 if(prefccs!=prenewfccs and not skipchecksum):
1391 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1392 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1393 return False;
1394 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1395 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1396 if(abs(fseeknextasnum)==0):
1397 pass;
1398 fp.seek(fseeknextasnum, 1);
1399 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1400 fseeknextasnum = int(prefseeknextfile);
1401 if(abs(fseeknextasnum)==0):
1402 pass;
1403 fp.seek(fseeknextasnum, 1);
1404 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1405 fseeknextasnum = int(prefseeknextfile);
1406 if(abs(fseeknextasnum)==0):
1407 pass;
1408 fp.seek(fseeknextasnum, 0);
1409 else:
1410 return False;
1411 il = il + 1;
1412 realidnum = 0;
1413 countnum = seekstart;
1414 while(countnum < seekend):
1415 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1416 if(len(HeaderOut)==0):
1417 break;
1418 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1419 catlist['ffilelist'].append(HeaderOut);
1420 countnum = countnum + 1;
1421 realidnum = realidnum + 1;
1422 return catlist;
1423
1424 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1425 formatspecs = FormatSpecsListToDict(formatspecs);
1426 delimiter = formatspecs['format_delimiter'];
1427 curloc = fp.tell();
1428 if(curloc>0):
1429 fp.seek(0, 0);
1430 catheader = ReadFileHeaderData(fp, 4, delimiter);
1431 if(curloc>0):
1432 fp.seek(curloc, 0);
1433 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1434 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1435 if(not headercheck and not skipchecksum):
1436 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1437 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1438 return False;
1439 catstring = catheader[0];
1440 catversion = re.findall(r"([\d]+)", catstring);
1441 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1442 fprenumfiles = catheader[1];
1443 fnumfiles = int(fprenumfiles, 16);
1444 fprechecksumtype = catheader[2];
1445 fprechecksum = catheader[3];
1446 catlist = [];
1447 if(seekstart<0 and seekstart>fnumfiles):
1448 seekstart = 0;
1449 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1450 seekend = fnumfiles;
1451 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1452 seekend = fnumfiles - abs(seekend);
1453 if(seekstart>0):
1454 il = 0;
1455 while(il < seekstart):
1456 prefhstart = fp.tell();
1457 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1458 if(len(preheaderdata)==0):
1459 break;
1460 prefsize = int(preheaderdata[5], 16);
1461 prefcompression = preheaderdata[12];
1462 prefcsize = int(preheaderdata[13], 16);
1463 prefseeknextfile = HeaderOut[25];
1464 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1465 prefcs = preheaderdata[-2];
1466 if(prefcs!=prenewfcs and not skipchecksum):
1467 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1468 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1469 return False;
1470 valid_archive = False;
1471 invalid_archive = True;
1472 prefhend = fp.tell() - 1;
1473 prefcontentstart = fp.tell();
1474 prefcontents = "";
1475 pyhascontents = False;
1476 if(prefsize>0):
1477 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1478 prefcontents = catfp.read(prefsize);
1479 else:
1480 prefcontents = catfp.read(prefcsize);
1481 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1482 prefccs = preheaderdata[-1];
1483 pyhascontents = True;
1484 if(prefccs!=prenewfccs and not skipchecksum):
1485 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1486 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1487 return False;
1488 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1489 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1490 if(abs(fseeknextasnum)==0):
1491 pass;
1492 catfp.seek(fseeknextasnum, 1);
1493 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1494 fseeknextasnum = int(prefseeknextfile);
1495 if(abs(fseeknextasnum)==0):
1496 pass;
1497 catfp.seek(fseeknextasnum, 1);
1498 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1499 fseeknextasnum = int(prefseeknextfile);
1500 if(abs(fseeknextasnum)==0):
1501 pass;
1502 catfp.seek(fseeknextasnum, 0);
1503 else:
1504 return False;
1505 il = il + 1;
1506 realidnum = 0;
1507 countnum = seekstart;
1508 while(countnum < seekend):
1509 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1510 if(len(HeaderOut)==0):
1511 break;
1512 catlist.append(HeaderOut);
1513 countnum = countnum + 1;
1514 realidnum = realidnum + 1;
1515 return catlist;
1516
1517 def ReadInFileBySizeWithContentToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1518 formatspecs = FormatSpecsListToDict(formatspecs);
1519 delimiter = formatspecs['format_delimiter'];
1520 if(hasattr(infile, "read") or hasattr(infile, "write")):
1521 fp = infile;
1522 fp.seek(0, 0);
1523 fp = UncompressArchiveFile(fp, formatspecs);
1524 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1525 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1526 return False;
1527 if(not fp):
1528 return False;
1529 fp.seek(0, 0);
1530 elif(infile=="-"):
1531 fp = BytesIO();
1532 if(hasattr(sys.stdin, "buffer")):
1533 shutil.copyfileobj(sys.stdin.buffer, fp);
1534 else:
1535 shutil.copyfileobj(sys.stdin, fp);
1536 fp.seek(0, 0);
1537 fp = UncompressArchiveFile(fp, formatspecs);
1538 if(not fp):
1539 return False;
1540 fp.seek(0, 0);
1541 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1542 fp = download_file_from_internet_file(infile);
1543 fp = UncompressArchiveFile(fp, formatspecs);
1544 fp.seek(0, 0);
1545 if(not fp):
1546 return False;
1547 fp.seek(0, 0);
1548 else:
1549 infile = RemoveWindowsPath(infile);
1550 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1551 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1552 return False;
1553 compresscheck = CheckCompressionType(infile, formatspecs, True);
1554 if(not compresscheck):
1555 fextname = os.path.splitext(infile)[1];
1556 if(fextname==".gz"):
1557 compresscheck = "gzip";
1558 elif(fextname==".bz2"):
1559 compresscheck = "bzip2";
1560 elif(fextname==".zst"):
1561 compresscheck = "zstd";
1562 elif(fextname==".lz4" or fextname==".clz4"):
1563 compresscheck = "lz4";
1564 elif(fextname==".lzo" or fextname==".lzop"):
1565 compresscheck = "lzo";
1566 elif(fextname==".lzma" or fextname==".xz"):
1567 compresscheck = "lzma";
1568 else:
1569 return False;
1570 if(not compresscheck):
1571 return False;
1572 fp = UncompressFile(infile, formatspecs, "rb");
1573 return ReadFileDataBySizeWithContentToArray(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1574
1575 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1576 formatspecs = FormatSpecsListToDict(formatspecs);
1577 delimiter = formatspecs['format_delimiter'];
1578 if(hasattr(infile, "read") or hasattr(infile, "write")):
1579 fp = infile;
1580 fp.seek(0, 0);
1581 fp = UncompressArchiveFile(fp, formatspecs);
1582 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1583 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1584 return False;
1585 if(not fp):
1586 return False;
1587 fp.seek(0, 0);
1588 elif(infile=="-"):
1589 fp = BytesIO();
1590 if(hasattr(sys.stdin, "buffer")):
1591 shutil.copyfileobj(sys.stdin.buffer, fp);
1592 else:
1593 shutil.copyfileobj(sys.stdin, fp);
1594 fp.seek(0, 0);
1595 fp = UncompressArchiveFile(fp, formatspecs);
1596 if(not fp):
1597 return False;
1598 fp.seek(0, 0);
1599 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1600 fp = download_file_from_internet_file(infile);
1601 fp = UncompressArchiveFile(fp, formatspecs);
1602 fp.seek(0, 0);
1603 if(not fp):
1604 return False;
1605 fp.seek(0, 0);
1606 else:
1607 infile = RemoveWindowsPath(infile);
1608 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1609 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1610 return False;
1611 compresscheck = CheckCompressionType(infile, formatspecs, True);
1612 if(not compresscheck):
1613 fextname = os.path.splitext(infile)[1];
1614 if(fextname==".gz"):
1615 compresscheck = "gzip";
1616 elif(fextname==".bz2"):
1617 compresscheck = "bzip2";
1618 elif(fextname==".zst"):
1619 compresscheck = "zstd";
1620 elif(fextname==".lz4" or fextname==".clz4"):
1621 compresscheck = "lz4";
1622 elif(fextname==".lzo" or fextname==".lzop"):
1623 compresscheck = "lzo";
1624 elif(fextname==".lzma" or fextname==".xz"):
1625 compresscheck = "lzma";
1626 else:
1627 return False;
1628 if(not compresscheck):
1629 return False;
1630 fp = UncompressFile(infile, formatspecs, "rb");
1631 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1632
1633 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1634 outdata = str(indata) + delimiter;
1635 return outdata;
1636
1637 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1638 outdata = "";
1639 inum = 0;
1640 il = len(indata);
1641 while(inum < il):
1642 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1643 inum = inum + 1;
1644 return outdata;
1645
1646 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1647 formatspecs = FormatSpecsListToDict(formatspecs);
1648 delimiter = formatspecs['format_delimiter'];
1649 catver = formatspecs['format_ver'];
1650 fileheaderver = str(int(catver.replace(".", "")));
1651 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1652 fp.write(fileheader.encode('UTF-8'));
1653 fnumfiles = format(int(numfiles), 'x').lower();
1654 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1655 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1656 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1657 fp.write(fnumfilesa.encode('UTF-8'));
1658 try:
1659 fp.flush();
1660 os.fsync(fp.fileno());
1661 except io.UnsupportedOperation:
1662 pass;
1663 except AttributeError:
1664 pass;
1665 except OSError as e:
1666 pass;
1667 return fp;
1668
1669 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1670 formatspecs = FormatSpecsListToDict(formatspecs);
1671 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1672 return fp;
1673
1674 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1675 formatspecs = FormatSpecsListToDict(formatspecs);
1676 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1677 if(os.path.exists(outfile)):
1678 try:
1679 os.unlink(outfile);
1680 except OSError as e:
1681 pass;
1682 if(outfile=="-"):
1683 verbose = False;
1684 catfpfp = BytesIO();
1685 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1686 catfp = outfile;
1687 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1688 catfp = BytesIO();
1689 else:
1690 fbasename = os.path.splitext(outfile)[0];
1691 fextname = os.path.splitext(outfile)[1];
1692 if(not compresswholefile and fextname in outextlistwd):
1693 compresswholefile = True;
1694 catfp = CompressOpenFile(outfile, True, compressionlevel);
1695 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1696 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1697 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1698 try:
1699 catfp.flush();
1700 os.fsync(catfp.fileno());
1701 except io.UnsupportedOperation:
1702 pass;
1703 except AttributeError:
1704 pass;
1705 except OSError as e:
1706 pass;
1707 if(outfile=="-"):
1708 catfp.seek(0, 0);
1709 if(hasattr(sys.stdout, "buffer")):
1710 shutil.copyfileobj(catfp, sys.stdout.buffer);
1711 else:
1712 shutil.copyfileobj(catfp, sys.stdout);
1713 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1714 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1715 catfp.seek(0, 0);
1716 upload_file_to_internet_file(catfp, outfile);
1717 if(returnfp):
1718 catfp.seek(0, 0);
1719 return catfp;
1720 else:
1721 catfp.close();
1722 return True;
1723
1724 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1725 formatspecs = FormatSpecsListToDict(formatspecs);
1726 extrafields = format(len(extradata), 'x').lower();
1727 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1728 if(len(extradata)>0):
1729 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1730 extrasizelen = format(len(extrasizestr), 'x').lower();
1731 catoutlen = len(filevalues) + len(extradata) + 5;
1732 catoutlenhex = format(catoutlen, 'x').lower();
1733 catoutlist = filevalues;
1734 catoutlist.insert(0, catoutlenhex);
1735 catoutlist.append(extrasizelen);
1736 catoutlist.append(extrafields);
1737 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1738 if(len(extradata)>0):
1739 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1740 if(len(filecontent)==0):
1741 checksumlist = [checksumtype, "none"];
1742 else:
1743 checksumlist = [checksumtype, checksumtype];
1744 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
1745 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1746 if(len(filecontent)==0):
1747 catfilecontentcshex = GetFileChecksum(filecontent, "none", False, formatspecs);
1748 else:
1749 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1750 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1751 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1752 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1753 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1754 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1755 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1756 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1757 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1758 fp.write(catfileout);
1759 try:
1760 fp.flush();
1761 os.fsync(fp.fileno());
1762 except io.UnsupportedOperation:
1763 pass;
1764 except AttributeError:
1765 pass;
1766 except OSError as e:
1767 pass;
1768 return fp;
1769
1770 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1771 formatspecs = FormatSpecsListToDict(formatspecs);
1772 advancedlist = formatspecs['use_advanced_list'];
1773 altinode = formatspecs['use_alt_inode'];
1774 if(verbose):
1775 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1776 if(infiles=="-"):
1777 for line in sys.stdin:
1778 infilelist.append(line.strip());
1779 infilelist = list(filter(None, infilelist));
1780 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1781 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1782 return False;
1783 with UncompressFile(infiles, formatspecs, "r") as finfile:
1784 for line in finfile:
1785 infilelist.append(line.strip());
1786 infilelist = list(filter(None, infilelist));
1787 else:
1788 if(isinstance(infiles, (list, tuple, ))):
1789 infilelist = list(filter(None, infiles));
1790 elif(isinstance(infiles, (str, ))):
1791 infilelist = list(filter(None, [infiles]));
1792 if(advancedlist):
1793 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1794 else:
1795 GetDirList = ListDir(infilelist, followlink, False);
1796 if(not GetDirList):
1797 return False;
1798 curinode = 0;
1799 curfid = 0;
1800 inodelist = [];
1801 inodetofile = {};
1802 filetoinode = {};
1803 inodetocatinode = {};
1804 numfiles = int(len(GetDirList));
1805 fnumfiles = format(numfiles, 'x').lower();
1806 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1807 for curfname in GetDirList:
1808 if(re.findall(r"^[.|/]", curfname)):
1809 fname = curfname;
1810 else:
1811 fname = "./"+curfname;
1812 if(verbose):
1813 VerbosePrintOut(fname);
1814 if(not followlink or followlink is None):
1815 fstatinfo = os.lstat(fname);
1816 else:
1817 fstatinfo = os.stat(fname);
1818 fpremode = fstatinfo.st_mode;
1819 finode = fstatinfo.st_ino;
1820 flinkcount = fstatinfo.st_nlink;
1821 ftype = 0;
1822 if(stat.S_ISREG(fpremode)):
1823 ftype = 0;
1824 elif(stat.S_ISLNK(fpremode)):
1825 ftype = 2;
1826 elif(stat.S_ISCHR(fpremode)):
1827 ftype = 3;
1828 elif(stat.S_ISBLK(fpremode)):
1829 ftype = 4;
1830 elif(stat.S_ISDIR(fpremode)):
1831 ftype = 5;
1832 elif(stat.S_ISFIFO(fpremode)):
1833 ftype = 6;
1834 elif(stat.S_ISSOCK(fpremode)):
1835 ftype = 8;
1836 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1837 ftype = 9;
1838 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1839 ftype = 10;
1840 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1841 ftype = 11;
1842 else:
1843 ftype = 0;
1844 flinkname = "";
1845 fcurfid = format(int(curfid), 'x').lower();
1846 if(not followlink and finode!=0):
1847 if(ftype!=1):
1848 if(finode in inodelist):
1849 ftype = 1;
1850 flinkname = inodetofile[finode];
1851 if(altinode):
1852 fcurinode = format(int(finode), 'x').lower();
1853 else:
1854 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1855 if(finode not in inodelist):
1856 inodelist.append(finode);
1857 inodetofile.update({finode: fname});
1858 inodetocatinode.update({finode: curinode});
1859 if(altinode):
1860 fcurinode = format(int(finode), 'x').lower();
1861 else:
1862 fcurinode = format(int(curinode), 'x').lower();
1863 curinode = curinode + 1;
1864 else:
1865 fcurinode = format(int(curinode), 'x').lower();
1866 curinode = curinode + 1;
1867 curfid = curfid + 1;
1868 if(ftype==2):
1869 flinkname = os.readlink(fname);
1870 fdev = fstatinfo.st_dev;
1871 getfdev = GetDevMajorMinor(fdev);
1872 fdev_minor = getfdev[0];
1873 fdev_major = getfdev[1];
1874 frdev = fstatinfo.st_dev;
1875 if(hasattr(fstatinfo, "st_rdev")):
1876 frdev = fstatinfo.st_rdev;
1877 else:
1878 frdev = fstatinfo.st_dev;
1879 getfrdev = GetDevMajorMinor(frdev);
1880 frdev_minor = getfrdev[0];
1881 frdev_major = getfrdev[1];
1882 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1883 fsize = format(int("0"), 'x').lower();
1884 elif(ftype==0 or ftype==7):
1885 fsize = format(int(fstatinfo.st_size), 'x').lower();
1886 else:
1887 fsize = format(int(fstatinfo.st_size)).lower();
1888 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1889 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1890 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1891 if(hasattr(fstatinfo, "st_birthtime")):
1892 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1893 else:
1894 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1895 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1896 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1897 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1898 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1899 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1900 funame = "";
1901 try:
1902 import pwd;
1903 try:
1904 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1905 funame = userinfo.pw_name;
1906 except KeyError:
1907 funame = "";
1908 except ImportError:
1909 funame = "";
1910 fgname = "";
1911 try:
1912 import grp;
1913 try:
1914 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1915 fgname = groupinfo.gr_name;
1916 except KeyError:
1917 fgname = "";
1918 except ImportError:
1919 fgname = "";
1920 fdev_minor = format(int(fdev_minor), 'x').lower();
1921 fdev_major = format(int(fdev_major), 'x').lower();
1922 frdev_minor = format(int(frdev_minor), 'x').lower();
1923 frdev_major = format(int(frdev_major), 'x').lower();
1924 finode = format(int(finode), 'x').lower();
1925 flinkcount = format(int(flinkcount), 'x').lower();
1926 if(hasattr(fstatinfo, "st_file_attributes")):
1927 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1928 else:
1929 fwinattributes = format(int(0), 'x').lower();
1930 fcompression = "";
1931 fcsize = format(int(0), 'x').lower();
1932 fcontents = BytesIO();
1933 chunk_size = 1024;
1934 if(ftype==0 or ftype==7):
1935 with open(fname, "rb") as fpc:
1936 shutil.copyfileobj(fpc, fcontents);
1937 if(not compresswholefile):
1938 fcontents.seek(0, 2);
1939 ucfsize = fcontents.tell();
1940 fcontents.seek(0, 0);
1941 if(compression=="auto"):
1942 ilsize = len(compressionlistalt);
1943 ilmin = 0;
1944 ilcsize = [];
1945 while(ilmin < ilsize):
1946 cfcontents = BytesIO();
1947 shutil.copyfileobj(fcontents, cfcontents);
1948 fcontents.seek(0, 0);
1949 cfcontents.seek(0, 0);
1950 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1951 if(cfcontents):
1952 cfcontents.seek(0, 2);
1953 ilcsize.append(cfcontents.tell());
1954 cfcontents.close();
1955 else:
1956 try:
1957 ilcsize.append(sys.maxint);
1958 except AttributeError:
1959 ilcsize.append(sys.maxsize);
1960 ilmin = ilmin + 1;
1961 ilcmin = ilcsize.index(min(ilcsize));
1962 compression = compressionlistalt[ilcmin];
1963 fcontents.seek(0, 0);
1964 cfcontents = BytesIO();
1965 shutil.copyfileobj(fcontents, cfcontents);
1966 cfcontents.seek(0, 0);
1967 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1968 cfcontents.seek(0, 2);
1969 cfsize = cfcontents.tell();
1970 if(ucfsize > cfsize):
1971 fcsize = format(int(cfsize), 'x').lower();
1972 fcompression = compression;
1973 fcontents.close();
1974 fcontents = cfcontents;
1975 if(followlink and (ftype==1 or ftype==2)):
1976 flstatinfo = os.stat(flinkname);
1977 with open(flinkname, "rb") as fpc:
1978 shutil.copyfileobj(fpc, fcontents);
1979 if(not compresswholefile):
1980 fcontents.seek(0, 2);
1981 ucfsize = fcontents.tell();
1982 fcontents.seek(0, 0);
1983 if(compression=="auto"):
1984 ilsize = len(compressionlistalt);
1985 ilmin = 0;
1986 ilcsize = [];
1987 while(ilmin < ilsize):
1988 cfcontents = BytesIO();
1989 shutil.copyfileobj(fcontents, cfcontents);
1990 fcontents.seek(0, 0);
1991 cfcontents.seek(0, 0);
1992 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1993 if(cfcontents):
1994 cfcontents.seek(0, 2);
1995 ilcsize.append(cfcontents.tell());
1996 cfcontents.close();
1997 else:
1998 try:
1999 ilcsize.append(sys.maxint);
2000 except AttributeError:
2001 ilcsize.append(sys.maxsize);
2002 ilmin = ilmin + 1;
2003 ilcmin = ilcsize.index(min(ilcsize));
2004 compression = compressionlistalt[ilcmin];
2005 fcontents.seek(0, 0);
2006 cfcontents = BytesIO();
2007 shutil.copyfileobj(fcontents, cfcontents);
2008 cfcontents.seek(0, 0);
2009 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2010 cfcontents.seek(0, 2);
2011 cfsize = cfcontents.tell();
2012 if(ucfsize > cfsize):
2013 fcsize = format(int(cfsize), 'x').lower();
2014 fcompression = compression;
2015 fcontents.close();
2016 fcontents = cfcontents;
2017 if(fcompression=="none"):
2018 fcompression = "";
2019 fcontents.seek(0, 0);
2020 ftypehex = format(ftype, 'x').lower();
2021 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
2022 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2023 if(numfiles>0):
2024 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2025 fp.seek(0, 0);
2026 return fp;
2027
2028 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2029 formatspecs = FormatSpecsListToDict(formatspecs);
2030 if(verbose):
2031 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2032 GetDirList = inlist;
2033 if(not GetDirList):
2034 return False;
2035 curinode = 0;
2036 curfid = 0;
2037 inodelist = [];
2038 inodetofile = {};
2039 filetoinode = {};
2040 inodetocatinode = {};
2041 numfiles = int(len(GetDirList));
2042 fnumfiles = format(numfiles, 'x').lower();
2043 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
2044 for curfname in GetDirList:
2045 ftype = format(curfname[0], 'x').lower();
2046 if(re.findall(r"^[.|/]", curfname[1])):
2047 fname = curfname[1];
2048 else:
2049 fname = "./"+curfname[1];
2050 fbasedir = os.path.dirname(fname);
2051 flinkname = curfname[2];
2052 fsize = format(curfname[3], 'x').lower();
2053 fatime = format(curfname[4], 'x').lower();
2054 fmtime = format(curfname[5], 'x').lower();
2055 fctime = format(curfname[6], 'x').lower();
2056 fbtime = format(curfname[7], 'x').lower();
2057 fmode = format(curfname[8], 'x').lower();
2058 fwinattributes = format(curfname[9], 'x').lower();
2059 fcompression = curfname[10];
2060 fcsize = format(curfname[11], 'x').lower();
2061 fuid = format(curfname[12], 'x').lower();
2062 funame = curfname[13];
2063 fgid = format(curfname[14], 'x').lower();
2064 fgname = curfname[15];
2065 fid = format(curfname[16], 'x').lower();
2066 finode = format(curfname[17], 'x').lower();
2067 flinkcount = format(curfname[18], 'x').lower();
2068 fdev_minor = format(curfname[19], 'x').lower();
2069 fdev_major = format(curfname[20], 'x').lower();
2070 frdev_minor = format(curfname[21], 'x').lower();
2071 frdev_major = format(curfname[22], 'x').lower();
2072 fseeknextfile = curfname[23];
2073 extradata = curfname[24];
2074 fheaderchecksumtype = curfname[25];
2075 fcontentchecksumtype = curfname[26];
2076 fcontents = curfname[27];
2077 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
2078 fcontents.seek(0, 0);
2079 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2080 if(numfiles>0):
2081 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2082 return fp;
2083
2084 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2085 formatspecs = FormatSpecsListToDict(formatspecs);
2086 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2087 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
2088
2089 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2090 formatspecs = FormatSpecsListToDict(formatspecs);
2091 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2092 if(os.path.exists(outfile)):
2093 try:
2094 os.unlink(outfile);
2095 except OSError as e:
2096 pass;
2097 if(outfile=="-"):
2098 verbose = False;
2099 catfpfp = BytesIO();
2100 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2101 catfp = outfile;
2102 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2103 catfp = BytesIO();
2104 else:
2105 fbasename = os.path.splitext(outfile)[0];
2106 fextname = os.path.splitext(outfile)[1];
2107 if(not compresswholefile and fextname in outextlistwd):
2108 compresswholefile = True;
2109 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2110 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2111 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2112 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2113 try:
2114 catfp.flush();
2115 os.fsync(catfp.fileno());
2116 except io.UnsupportedOperation:
2117 pass;
2118 except AttributeError:
2119 pass;
2120 except OSError as e:
2121 pass;
2122 if(outfile=="-"):
2123 catfp.seek(0, 0);
2124 if(hasattr(sys.stdout, "buffer")):
2125 shutil.copyfileobj(catfp, sys.stdout.buffer);
2126 else:
2127 shutil.copyfileobj(catfp, sys.stdout);
2128 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2129 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2130 catfp.seek(0, 0);
2131 upload_file_to_internet_file(catfp, outfile);
2132 if(returnfp):
2133 catfp.seek(0, 0);
2134 return catfp;
2135 else:
2136 catfp.close();
2137 return True;
2138
2139 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2140 formatspecs = FormatSpecsListToDict(formatspecs);
2141 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2142 if(os.path.exists(outfile)):
2143 try:
2144 os.unlink(outfile);
2145 except OSError as e:
2146 pass;
2147 if(outfile=="-"):
2148 verbose = False;
2149 catfpfp = BytesIO();
2150 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2151 catfp = outfile;
2152 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2153 catfp = BytesIO();
2154 else:
2155 fbasename = os.path.splitext(outfile)[0];
2156 fextname = os.path.splitext(outfile)[1];
2157 if(not compresswholefile and fextname in outextlistwd):
2158 compresswholefile = True;
2159 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2160 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2161 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2162 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2163 try:
2164 catfp.flush();
2165 os.fsync(catfp.fileno());
2166 except io.UnsupportedOperation:
2167 pass;
2168 except AttributeError:
2169 pass;
2170 except OSError as e:
2171 pass;
2172 if(outfile=="-"):
2173 catfp.seek(0, 0);
2174 if(hasattr(sys.stdout, "buffer")):
2175 shutil.copyfileobj(catfp, sys.stdout.buffer);
2176 else:
2177 shutil.copyfileobj(catfp, sys.stdout);
2178 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2179 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2180 catfp.seek(0, 0);
2181 upload_file_to_internet_file(catfp, outfile);
2182 if(returnfp):
2183 catfp.seek(0, 0);
2184 return catfp;
2185 else:
2186 catfp.close();
2187 return True;
2188
2189 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2190 formatspecs = FormatSpecsListToDict(formatspecs);
2191 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2192 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2193
2194 def PrintPermissionString(fchmode, ftype):
2195 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2196 permissionstr = "";
2197 for fmodval in str(oct(fchmode))[-3:]:
2198 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2199 if(ftype==0 or ftype==7):
2200 permissionstr = "-" + permissionstr;
2201 if(ftype==1):
2202 permissionstr = "h" + permissionstr;
2203 if(ftype==2):
2204 permissionstr = "l" + permissionstr;
2205 if(ftype==3):
2206 permissionstr = "c" + permissionstr;
2207 if(ftype==4):
2208 permissionstr = "b" + permissionstr;
2209 if(ftype==5):
2210 permissionstr = "d" + permissionstr;
2211 if(ftype==6):
2212 permissionstr = "f" + permissionstr;
2213 if(ftype==8):
2214 permissionstr = "D" + permissionstr;
2215 if(ftype==9):
2216 permissionstr = "p" + permissionstr;
2217 if(ftype==10):
2218 permissionstr = "w" + permissionstr;
2219 try:
2220 permissionoutstr = stat.filemode(fchmode);
2221 except AttributeError:
2222 permissionoutstr = permissionstr;
2223 except KeyError:
2224 permissionoutstr = permissionstr;
2225 return permissionoutstr;
2226
2227 def PrintPermissionStringAlt(fchmode, ftype):
2228 permissions = {
2229 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2230 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2231 };
2232 # Translate file mode into permission string
2233 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2234 # Append file type indicator
2235 type_indicators = {
2236 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2237 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2238 };
2239 file_type = type_indicators.get(ftype, '-');
2240 permissionstr = file_type + permissionstr;
2241 try:
2242 permissionoutstr = stat.filemode(fchmode);
2243 except AttributeError:
2244 permissionoutstr = permissionstr;
2245 return permissionoutstr;
2246
2247 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2248 formatspecs = FormatSpecsListToDict(formatspecs);
2249 if(hasattr(infile, "read") or hasattr(infile, "write")):
2250 catfp = infile;
2251 else:
2252 try:
2253 catfp = open(infile, "rb");
2254 except FileNotFoundError:
2255 return False;
2256 catfp.seek(0, 0);
2257 prefp = catfp.read(2);
2258 filetype = False;
2259 if(prefp==binascii.unhexlify("1f8b")):
2260 filetype = "gzip";
2261 catfp.seek(0, 0);
2262 prefp = catfp.read(3);
2263 if(prefp==binascii.unhexlify("425a68")):
2264 filetype = "bzip2";
2265 if(prefp==binascii.unhexlify("5d0000")):
2266 filetype = "lzma";
2267 catfp.seek(0, 0);
2268 prefp = catfp.read(4);
2269 if(prefp==binascii.unhexlify("28b52ffd")):
2270 filetype = "zstd";
2271 if(prefp==binascii.unhexlify("04224d18")):
2272 filetype = "lz4";
2273 if(prefp==binascii.unhexlify("504B0304")):
2274 filetype = "zipfile";
2275 catfp.seek(0, 0);
2276 prefp = catfp.read(5);
2277 if(prefp==binascii.unhexlify("7573746172")):
2278 filetype = "tarfile";
2279 catfp.seek(0, 0);
2280 prefp = catfp.read(6);
2281 if(prefp==binascii.unhexlify("fd377a585a00")):
2282 filetype = "lzma";
2283 if(prefp==binascii.unhexlify("377abcaf271c")):
2284 filetype = "7zipfile";
2285 catfp.seek(0, 0);
2286 prefp = catfp.read(7);
2287 if(prefp==binascii.unhexlify("526172211a0700")):
2288 filetype = "rarfile";
2289 if(prefp==binascii.unhexlify("43617446696c65")):
2290 filetype = "catfile";
2291 catfp.seek(0, 0);
2292 prefp = catfp.read(8);
2293 if(prefp==binascii.unhexlify("526172211a070100")):
2294 filetype = "rarfile";
2295 catfp.seek(0, 0);
2296 prefp = catfp.read(formatspecs['format_len']);
2297 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2298 filetype = formatspecs['format_lower'];
2299 catfp.seek(0, 0);
2300 prefp = catfp.read(9);
2301 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2302 filetype = "lzo";
2303 catfp.seek(0, 0);
2304 prefp = catfp.read(10);
2305 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2306 filetype = "tarfile";
2307 catfp.seek(0, 0);
2308 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2309 if(TarFileCheck(catfp)):
2310 filetype = "tarfile";
2311 if(not filetype):
2312 if(TarFileCheck(catfp)):
2313 filetype = "tarfile";
2314 elif(zipfile.is_zipfile(catfp)):
2315 filetype = "zipfile";
2316 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2317 filetype = "rarile";
2318 else:
2319 filetype = False;
2320 catfp.seek(0, 0);
2321 if(closefp):
2322 catfp.close();
2323 return filetype;
2324
2325 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2326 formatspecs = FormatSpecsListToDict(formatspecs);
2327 try:
2328 instringsfile = BytesIO(instring);
2329 except TypeError:
2330 instringsfile = BytesIO(instring.encode("UTF-8"));
2331 return CheckCompressionType(instringsfile, formatspecs, closefp);
2332
2333 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2334 formatspecs = FormatSpecsListToDict(formatspecs);
2335 compresscheck = CheckCompressionType(fp, formatspecs, False);
2336 if(compresscheck=="gzip" or compresscheck=="gz"):
2337 return archivefile_gzip_mimetype;
2338 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2339 return archivefile_bzip2_mimetype;
2340 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2341 return archivefile_zstandard_mimetype;
2342 if(compresscheck=="lz4"):
2343 return archivefile_lz4_mimetype;
2344 if(compresscheck=="lzo" or compresscheck=="lzop"):
2345 return archivefile_lzop_mimetype;
2346 if(compresscheck=="lzma"):
2347 return archivefile_lzma_mimetype;
2348 if(compresscheck=="xz"):
2349 return archivefile_xz_mimetype;
2350 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2351 return archivefile_cat_mimetype;
2352 if(not compresscheck):
2353 return False;
2354 return False;
2355
2356 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2357 formatspecs = FormatSpecsListToDict(formatspecs);
2358 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2359 return False;
2360 compresscheck = CheckCompressionType(fp, formatspecs, False);
2361 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2362 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2363 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2364 catfp = BytesIO();
2365 catfp.write(bz2.decompress(fp.read()));
2366 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2367 catfp = BytesIO();
2368 catfp.write(zstandard.decompress(fp.read()));
2369 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2370 catfp = BytesIO();
2371 catfp.write(lz4.frame.decompress(fp.read()));
2372 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2373 catfp = BytesIO();
2374 catfp.write(lzo.decompress(fp.read()));
2375 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2376 catfp = BytesIO();
2377 catfp.write(lzma.decompress(fp.read()));
2378 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2379 catfp = fp;
2380 if(not compresscheck):
2381 try:
2382 import lzma;
2383 except ImportError:
2384 try:
2385 from backports import lzma
2386 except ImportError:
2387 return False;
2388 catfp = BytesIO();
2389 with fp as fpcontent:
2390 try:
2391 catfp.write(lzma.decompress(fp.read()));
2392 except lzma.LZMAError:
2393 return False;
2394 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2395 fp.close();
2396 return catfp;
2397
2398 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2399
2400 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2401 formatspecs = FormatSpecsListToDict(formatspecs);
2402 compresscheck = CheckCompressionType(infile, formatspecs, False);
2403 if(sys.version_info[0]==2 and compresscheck):
2404 if(mode=="rt"):
2405 mode = "r";
2406 if(mode=="wt"):
2407 mode = "w";
2408 try:
2409 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2410 try:
2411 filefp = gzip.open(infile, mode, encoding="UTF-8");
2412 except (ValueError, TypeError) as e:
2413 filefp = gzip.open(infile, mode);
2414 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2415 try:
2416 filefp = bz2.open(infile, mode, encoding="UTF-8");
2417 except (ValueError, TypeError) as e:
2418 filefp = bz2.open(infile, mode);
2419 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2420 try:
2421 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2422 except (ValueError, TypeError) as e:
2423 filefp = zstandard.open(infile, mode);
2424 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2425 try:
2426 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2427 except (ValueError, TypeError) as e:
2428 filefp = lz4.frame.open(infile, mode);
2429 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2430 try:
2431 filefp = lzo.open(infile, mode, encoding="UTF-8");
2432 except (ValueError, TypeError) as e:
2433 filefp = lzo.open(infile, mode);
2434 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2435 try:
2436 filefp = lzma.open(infile, mode, encoding="UTF-8");
2437 except (ValueError, TypeError) as e:
2438 filefp = lzma.open(infile, mode);
2439 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2440 try:
2441 filefp = open(infile, mode, encoding="UTF-8");
2442 except (ValueError, TypeError) as e:
2443 filefp = open(infile, mode);
2444 if(not compresscheck):
2445 try:
2446 filefp = open(infile, mode, encoding="UTF-8");
2447 except (ValueError, TypeError) as e:
2448 filefp = open(infile, mode);
2449 except FileNotFoundError:
2450 return False;
2451 return filefp;
2452
2453 def UncompressString(infile):
2454 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2455 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2456 fileuz = gzip.decompress(infile);
2457 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2458 fileuz = bz2.decompress(infile);
2459 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2460 try:
2461 import zstandard;
2462 except ImportError:
2463 return False;
2464 fileuz = zstandard.decompress(infile);
2465 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2466 fileuz = lz4.frame.decompress(infile);
2467 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2468 fileuz = lzo.decompress(infile);
2469 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2470 fileuz = lzma.decompress(infile);
2471 if(not compresscheck):
2472 fileuz = infile;
2473 if(hasattr(fileuz, 'decode')):
2474 fileuz = fileuz.decode("UTF-8");
2475 return fileuz;
2476
2477 def UncompressStringAlt(infile):
2478 filefp = StringIO();
2479 outstring = UncompressString(infile);
2480 filefp.write(outstring);
2481 filefp.seek(0, 0);
2482 return filefp;
2483
2484 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2485 formatspecs = FormatSpecsListToDict(formatspecs);
2486 compresscheck = CheckCompressionType(infile, formatspecs, False);
2487 if(not compresscheck):
2488 fextname = os.path.splitext(infile)[1];
2489 if(fextname==".gz"):
2490 compresscheck = "gzip";
2491 elif(fextname==".bz2"):
2492 compresscheck = "bzip2";
2493 elif(fextname==".zst"):
2494 compresscheck = "zstd";
2495 elif(fextname==".lz4"):
2496 compresscheck = "lz4";
2497 elif(fextname==".lzo" or fextname==".lzop"):
2498 compresscheck = "lzo";
2499 elif(fextname==".lzma" or fextname==".xz"):
2500 compresscheck = "lzma";
2501 else:
2502 return False;
2503 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2504 if(TarFileCheck(infile)):
2505 filetype = "tarfile";
2506 if(not compresscheck):
2507 if(TarFileCheck(infile)):
2508 return "tarfile";
2509 elif(zipfile.is_zipfile(infile)):
2510 return "zipfile";
2511 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2512 return "rarile";
2513 else:
2514 return False;
2515 return False;
2516 if(compresscheck=="catfile"):
2517 return "catfile";
2518 if(compresscheck==formatspecs['format_lower']):
2519 return formatspecs['format_lower'];
2520 if(compresscheck=="tarfile"):
2521 return "tarfile";
2522 if(compresscheck=="zipfile"):
2523 return "zipfile";
2524 if(rarfile_support and compresscheck=="rarfile"):
2525 return "rarfile";
2526 if(py7zr_support and compresscheck=="7zipfile" and py7zr.is_7zfile(infile)):
2527 return "7zipfile";
2528 if(hasattr(infile, "read") or hasattr(infile, "write")):
2529 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2530 else:
2531 try:
2532 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2533 catfp = gzip.GzipFile(infile, "rb");
2534 elif(compresscheck=="bzip2" and compresscheck in compressionsupport):
2535 catfp = bz2.BZ2File(infile, "rb");
2536 elif(compresscheck=="lz4" and compresscheck in compressionsupport):
2537 catfp = lz4.frame.open(infile, "rb");
2538 elif(compresscheck=="zstd" and compresscheck in compressionsupport):
2539 catfp = zstandard.open(infile, "rb");
2540 elif((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2541 catfp = lzo.open(infile, "rb");
2542 elif((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2543 catfp = lzma.open(infile, "rb");
2544 else:
2545 catfp = open(infile, "rb");
2546 except FileNotFoundError:
2547 return False;
2548 filetype = False;
2549 prefp = catfp.read(5);
2550 if(prefp==binascii.unhexlify("7573746172")):
2551 filetype = "tarfile";
2552 catfp.seek(0, 0);
2553 prefp = catfp.read(7);
2554 if(prefp==binascii.unhexlify("43617446696c65")):
2555 filetype = "catfile";
2556 catfp.seek(0, 0);
2557 prefp = catfp.read(formatspecs['format_len']);
2558 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2559 filetype = formatspecs['format_lower'];
2560 catfp.seek(0, 0);
2561 prefp = catfp.read(10);
2562 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2563 filetype = "tarfile";
2564 catfp.seek(0, 0);
2565 if(closefp):
2566 catfp.close();
2567 return filetype;
2568
2569 def GZipCompress(data, compresslevel=9):
2570 if("gzip" not in compressionsupport):
2571 return False;
2572 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2573 tmpfp.close();
2574 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2575 tmpfp.write(data);
2576 tmpfp.close();
2577 try:
2578 catfp = open(tmpfp.name, "rb");
2579 except FileNotFoundError:
2580 return False;
2581 catdata = catfp.read();
2582 catfp.close();
2583 return catdata;
2584
2585 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2586 formatspecs = FormatSpecsListToDict(formatspecs);
2587 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2588 return False;
2589 fp.seek(0, 0);
2590 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2591 compression = "auto";
2592 if(compression not in compressionlist and compression is None):
2593 compression = "auto";
2594 if(compression=="gzip" and compression in compressionsupport):
2595 catfp = BytesIO();
2596 if(compressionlevel is None):
2597 compressionlevel = 9;
2598 else:
2599 compressionlevel = int(compressionlevel);
2600 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2601 if(compression=="bzip2" and compression in compressionsupport):
2602 catfp = BytesIO();
2603 if(compressionlevel is None):
2604 compressionlevel = 9;
2605 else:
2606 compressionlevel = int(compressionlevel);
2607 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2608 if(compression=="lz4" and compression in compressionsupport):
2609 catfp = BytesIO();
2610 if(compressionlevel is None):
2611 compressionlevel = 9;
2612 else:
2613 compressionlevel = int(compressionlevel);
2614 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2615 if((compression=="lzo" or compression=="lzop") and compression in compressionsupport):
2616 catfp = BytesIO();
2617 if(compressionlevel is None):
2618 compressionlevel = 9;
2619 else:
2620 compressionlevel = int(compressionlevel);
2621 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2622 if(compression=="zstd" and compression in compressionsupport):
2623 catfp = BytesIO();
2624 if(compressionlevel is None):
2625 compressionlevel = 10;
2626 else:
2627 compressionlevel = int(compressionlevel);
2628 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2629 if(compression=="lzma" and compression in compressionsupport):
2630 catfp = BytesIO();
2631 if(compressionlevel is None):
2632 compressionlevel = 9;
2633 else:
2634 compressionlevel = int(compressionlevel);
2635 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2636 if(compression=="xz" and compression in compressionsupport):
2637 catfp = BytesIO();
2638 if(compressionlevel is None):
2639 compressionlevel = 9;
2640 else:
2641 compressionlevel = int(compressionlevel);
2642 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2643 if(compression=="auto" or compression is None):
2644 catfp = fp;
2645 catfp.seek(0, 0);
2646 return catfp;
2647
2648 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2649
2650 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2651 if(outfile is None):
2652 return False;
2653 fbasename = os.path.splitext(outfile)[0];
2654 fextname = os.path.splitext(outfile)[1];
2655 if(compressionlevel is None and fextname!=".zst"):
2656 compressionlevel = 9;
2657 elif(compressionlevel is None and fextname==".zst"):
2658 compressionlevel = 10;
2659 else:
2660 compressionlevel = int(compressionlevel);
2661 if(sys.version_info[0]==2):
2662 mode = "w";
2663 else:
2664 mode = "wb";
2665 try:
2666 if(fextname not in outextlistwd or not compressionenable):
2667 try:
2668 outfp = open(outfile, "wb", encoding="UTF-8");
2669 except (ValueError, TypeError) as e:
2670 outfp = open(outfile, "wb");
2671 elif(fextname==".gz" and "gzip" in compressionsupport):
2672 try:
2673 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2674 except (ValueError, TypeError) as e:
2675 outfp = gzip.open(outfile, mode, compressionlevel);
2676 elif(fextname==".bz2" and "bzip2" in compressionsupport):
2677 try:
2678 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2679 except (ValueError, TypeError) as e:
2680 outfp = bz2.open(outfile, mode, compressionlevel);
2681 elif(fextname==".zst" and "zstandard" in compressionsupport):
2682 try:
2683 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2684 except (ValueError, TypeError) as e:
2685 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2686 elif(fextname==".xz" and "xz" in compressionsupport):
2687 try:
2688 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2689 except (ValueError, TypeError) as e:
2690 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2691 elif(fextname==".lz4" and "lz4" in compressionsupport):
2692 try:
2693 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2694 except (ValueError, TypeError) as e:
2695 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2696 elif(fextname==".lzo" and "lzop" in compressionsupport):
2697 try:
2698 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2699 except (ValueError, TypeError) as e:
2700 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2701 elif(fextname==".lzma" and "lzma" in compressionsupport):
2702 try:
2703 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2704 except (ValueError, TypeError) as e:
2705 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2706 except FileNotFoundError:
2707 return False;
2708 return outfp;
2709
2710 def GetDevMajorMinor(fdev):
2711 retdev = [];
2712 if(hasattr(os, "minor")):
2713 retdev.append(os.minor(fdev));
2714 else:
2715 retdev.append(0);
2716 if(hasattr(os, "major")):
2717 retdev.append(os.major(fdev));
2718 else:
2719 retdev.append(0);
2720 return retdev;
2721
2722 def CheckSumSupport(checkfor, guaranteed=True):
2723 if(guaranteed):
2724 try:
2725 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2726 except AttributeError:
2727 hash_list = sorted(list(hashlib.algorithms));
2728 else:
2729 try:
2730 hash_list = sorted(list(hashlib.algorithms_available));
2731 except AttributeError:
2732 hash_list = sorted(list(hashlib.algorithms));
2733 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2734 if(checkfor in checklistout):
2735 return True;
2736 else:
2737 return False;
2738
2739 def CheckSumSupportAlt(checkfor, guaranteed=True):
2740 if(guaranteed):
2741 try:
2742 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2743 except AttributeError:
2744 hash_list = sorted(list(hashlib.algorithms));
2745 else:
2746 try:
2747 hash_list = sorted(list(hashlib.algorithms_available));
2748 except AttributeError:
2749 hash_list = sorted(list(hashlib.algorithms));
2750 checklistout = hash_list;
2751 if(checkfor in checklistout):
2752 return True;
2753 else:
2754 return False;
2755
2756 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2757 formatspecs = FormatSpecsListToDict(formatspecs);
2758 advancedlist = formatspecs['use_advanced_list'];
2759 altinode = formatspecs['use_alt_inode'];
2760 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2761 outfile = RemoveWindowsPath(outfile);
2762 checksumtype = checksumtype.lower();
2763 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2764 checksumtype="crc32";
2765 if(checksumtype=="none"):
2766 checksumtype = "";
2767 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2768 compression = "auto";
2769 if(compression not in compressionlist and compression is None):
2770 compression = "auto";
2771 if(verbose):
2772 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2773 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2774 if(os.path.exists(outfile)):
2775 try:
2776 os.unlink(outfile);
2777 except OSError as e:
2778 pass;
2779 if(outfile=="-"):
2780 verbose = False;
2781 catfp = BytesIO();
2782 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2783 catfp = outfile;
2784 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2785 catfp = BytesIO();
2786 else:
2787 fbasename = os.path.splitext(outfile)[0];
2788 fextname = os.path.splitext(outfile)[1];
2789 if(not compresswholefile and fextname in outextlistwd):
2790 compresswholefile = True;
2791 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2792 catver = formatspecs['format_ver'];
2793 fileheaderver = str(int(catver.replace(".", "")));
2794 infilelist = [];
2795 if(infiles=="-"):
2796 for line in sys.stdin:
2797 infilelist.append(line.strip());
2798 infilelist = list(filter(None, infilelist));
2799 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2800 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2801 return False;
2802 with UncompressFile(infiles, formatspecs, "r") as finfile:
2803 for line in finfile:
2804 infilelist.append(line.strip());
2805 infilelist = list(filter(None, infilelist));
2806 else:
2807 if(isinstance(infiles, (list, tuple, ))):
2808 infilelist = list(filter(None, infiles));
2809 elif(isinstance(infiles, (str, ))):
2810 infilelist = list(filter(None, [infiles]));
2811 if(advancedlist):
2812 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2813 else:
2814 GetDirList = ListDir(infilelist, followlink, False);
2815 if(not GetDirList):
2816 return False;
2817 curinode = 0;
2818 curfid = 0;
2819 inodelist = [];
2820 inodetofile = {};
2821 filetoinode = {};
2822 inodetocatinode = {};
2823 numfiles = int(len(GetDirList));
2824 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2825 for curfname in GetDirList:
2826 if(re.findall(r"^[.|/]", curfname)):
2827 fname = curfname;
2828 else:
2829 fname = "./"+curfname;
2830 if(verbose):
2831 VerbosePrintOut(fname);
2832 if(not followlink or followlink is None):
2833 fstatinfo = os.lstat(fname);
2834 else:
2835 fstatinfo = os.stat(fname);
2836 fpremode = fstatinfo.st_mode;
2837 finode = fstatinfo.st_ino;
2838 flinkcount = fstatinfo.st_nlink;
2839 ftype = 0;
2840 if(stat.S_ISREG(fpremode)):
2841 ftype = 0;
2842 elif(stat.S_ISLNK(fpremode)):
2843 ftype = 2;
2844 elif(stat.S_ISCHR(fpremode)):
2845 ftype = 3;
2846 elif(stat.S_ISBLK(fpremode)):
2847 ftype = 4;
2848 elif(stat.S_ISDIR(fpremode)):
2849 ftype = 5;
2850 elif(stat.S_ISFIFO(fpremode)):
2851 ftype = 6;
2852 elif(stat.S_ISSOCK(fpremode)):
2853 ftype = 8;
2854 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2855 ftype = 9;
2856 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2857 ftype = 10;
2858 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2859 ftype = 11;
2860 else:
2861 ftype = 0;
2862 flinkname = "";
2863 fcurfid = format(int(curfid), 'x').lower();
2864 if(not followlink and finode!=0):
2865 if(ftype!=1):
2866 if(finode in inodelist):
2867 ftype = 1;
2868 flinkname = inodetofile[finode];
2869 if(altinode):
2870 fcurinode = format(int(finode), 'x').lower();
2871 else:
2872 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2873 if(finode not in inodelist):
2874 inodelist.append(finode);
2875 inodetofile.update({finode: fname});
2876 inodetocatinode.update({finode: curinode});
2877 if(altinode):
2878 fcurinode = format(int(finode), 'x').lower();
2879 else:
2880 fcurinode = format(int(curinode), 'x').lower();
2881 curinode = curinode + 1;
2882 else:
2883 fcurinode = format(int(curinode), 'x').lower();
2884 curinode = curinode + 1;
2885 curfid = curfid + 1;
2886 if(ftype==2):
2887 flinkname = os.readlink(fname);
2888 fdev = fstatinfo.st_dev;
2889 getfdev = GetDevMajorMinor(fdev);
2890 fdev_minor = getfdev[0];
2891 fdev_major = getfdev[1];
2892 frdev = fstatinfo.st_dev;
2893 if(hasattr(fstatinfo, "st_rdev")):
2894 frdev = fstatinfo.st_rdev;
2895 else:
2896 frdev = fstatinfo.st_dev;
2897 getfrdev = GetDevMajorMinor(frdev);
2898 frdev_minor = getfrdev[0];
2899 frdev_major = getfrdev[1];
2900 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2901 fsize = format(int("0"), 'x').lower();
2902 elif(ftype==0 or ftype==7):
2903 fsize = format(int(fstatinfo.st_size), 'x').lower();
2904 else:
2905 fsize = format(int(fstatinfo.st_size)).lower();
2906 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2907 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2908 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2909 if(hasattr(fstatinfo, "st_birthtime")):
2910 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2911 else:
2912 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2913 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2914 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2915 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2916 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2917 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2918 funame = "";
2919 try:
2920 import pwd;
2921 try:
2922 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2923 funame = userinfo.pw_name;
2924 except KeyError:
2925 funame = "";
2926 except ImportError:
2927 funame = "";
2928 fgname = "";
2929 try:
2930 import grp;
2931 try:
2932 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2933 fgname = groupinfo.gr_name;
2934 except KeyError:
2935 fgname = "";
2936 except ImportError:
2937 fgname = "";
2938 fdev_minor = format(int(fdev_minor), 'x').lower();
2939 fdev_major = format(int(fdev_major), 'x').lower();
2940 frdev_minor = format(int(frdev_minor), 'x').lower();
2941 frdev_major = format(int(frdev_major), 'x').lower();
2942 finode = format(int(finode), 'x').lower();
2943 flinkcount = format(int(flinkcount), 'x').lower();
2944 if(hasattr(fstatinfo, "st_file_attributes")):
2945 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2946 else:
2947 fwinattributes = format(int(0), 'x').lower();
2948 fcompression = "";
2949 fcsize = format(int(0), 'x').lower();
2950 fcontents = BytesIO();
2951 if(ftype==0 or ftype==7):
2952 with open(fname, "rb") as fpc:
2953 shutil.copyfileobj(fpc, fcontents);
2954 if(not compresswholefile):
2955 fcontents.seek(0, 2);
2956 ucfsize = fcontents.tell();
2957 fcontents.seek(0, 0);
2958 if(compression=="auto"):
2959 ilsize = len(compressionlistalt);
2960 ilmin = 0;
2961 ilcsize = [];
2962 while(ilmin < ilsize):
2963 cfcontents = BytesIO();
2964 shutil.copyfileobj(fcontents, cfcontents);
2965 fcontents.seek(0, 0);
2966 cfcontents.seek(0, 0);
2967 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2968 if(cfcontents):
2969 cfcontents.seek(0, 2);
2970 ilcsize.append(cfcontents.tell());
2971 cfcontents.close();
2972 else:
2973 try:
2974 ilcsize.append(sys.maxint);
2975 except AttributeError:
2976 ilcsize.append(sys.maxsize);
2977 ilmin = ilmin + 1;
2978 ilcmin = ilcsize.index(min(ilcsize));
2979 compression = compressionlistalt[ilcmin];
2980 fcontents.seek(0, 0);
2981 cfcontents = BytesIO();
2982 shutil.copyfileobj(fcontents, cfcontents);
2983 cfcontents.seek(0, 0);
2984 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2985 cfcontents.seek(0, 2);
2986 cfsize = cfcontents.tell();
2987 if(ucfsize > cfsize):
2988 fcsize = format(int(cfsize), 'x').lower();
2989 fcompression = compression;
2990 fcontents.close();
2991 fcontents = cfcontents;
2992 if(fcompression=="none"):
2993 fcompression = "";
2994 if(followlink and (ftype==1 or ftype==2)):
2995 flstatinfo = os.stat(flinkname);
2996 with open(flinkname, "rb") as fpc:
2997 shutil.copyfileobj(fpc, fcontents);
2998 if(not compresswholefile):
2999 fcontents.seek(0, 2);
3000 ucfsize = fcontents.tell();
3001 fcontents.seek(0, 0);
3002 if(compression=="auto"):
3003 ilsize = len(compressionlistalt);
3004 ilmin = 0;
3005 ilcsize = [];
3006 while(ilmin < ilsize):
3007 cfcontents = BytesIO();
3008 shutil.copyfileobj(fcontents, cfcontents);
3009 fcontents.seek(0, 0);
3010 cfcontents.seek(0, 0);
3011 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3012 if(cfcontents):
3013 cfcontents.seek(0, 2);
3014 ilcsize.append(cfcontents.tell());
3015 cfcontents.close();
3016 else:
3017 try:
3018 ilcsize.append(sys.maxint);
3019 except AttributeError:
3020 ilcsize.append(sys.maxsize);
3021 ilmin = ilmin + 1;
3022 ilcmin = ilcsize.index(min(ilcsize));
3023 compression = compressionlistalt[ilcmin];
3024 fcontents.seek(0, 0);
3025 cfcontents = BytesIO();
3026 shutil.copyfileobj(fcontents, cfcontents);
3027 cfcontents.seek(0, 0);
3028 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3029 cfcontents.seek(0, 2);
3030 cfsize = cfcontents.tell();
3031 if(ucfsize > cfsize):
3032 fcsize = format(int(cfsize), 'x').lower();
3033 fcompression = compression;
3034 fcontents.close();
3035 fcontents = cfcontents;
3036 fcontents.seek(0, 0);
3037 ftypehex = format(ftype, 'x').lower();
3038 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3039 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3040 fcontents.close();
3041 if(numfiles>0):
3042 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3043 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3044 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3045 try:
3046 catfp.flush();
3047 os.fsync(catfp.fileno());
3048 except io.UnsupportedOperation:
3049 pass;
3050 except AttributeError:
3051 pass;
3052 except OSError as e:
3053 pass;
3054 if(outfile=="-"):
3055 catfp.seek(0, 0);
3056 if(hasattr(sys.stdout, "buffer")):
3057 shutil.copyfileobj(catfp, sys.stdout.buffer);
3058 else:
3059 shutil.copyfileobj(catfp, sys.stdout);
3060 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3061 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3062 catfp.seek(0, 0);
3063 upload_file_to_internet_file(catfp, outfile);
3064 if(returnfp):
3065 catfp.seek(0, 0);
3066 return catfp;
3067 else:
3068 catfp.close();
3069 return True;
3070
3071 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3072
3073 if(hasattr(shutil, "register_archive_format")):
3074 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3075 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3076 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3077
3078 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3079 formatspecs = FormatSpecsListToDict(formatspecs);
3080 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3081
3082 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3083
3084 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3085 formatspecs = FormatSpecsListToDict(formatspecs);
3086 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3087 outfile = RemoveWindowsPath(outfile);
3088 checksumtype = checksumtype.lower();
3089 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3090 checksumtype="crc32";
3091 if(checksumtype=="none"):
3092 checksumtype = "";
3093 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3094 compression = "auto";
3095 if(compression not in compressionlist and compression is None):
3096 compression = "auto";
3097 if(verbose):
3098 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3099 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3100 if(os.path.exists(outfile)):
3101 try:
3102 os.unlink(outfile);
3103 except OSError as e:
3104 pass;
3105 if(outfile=="-"):
3106 verbose = False;
3107 catfp = BytesIO();
3108 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3109 catfp = outfile;
3110 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3111 catfp = BytesIO();
3112 else:
3113 fbasename = os.path.splitext(outfile)[0];
3114 fextname = os.path.splitext(outfile)[1];
3115 if(not compresswholefile and fextname in outextlistwd):
3116 compresswholefile = True;
3117 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3118 catver = formatspecs['format_ver'];
3119 fileheaderver = str(int(catver.replace(".", "")));
3120 curinode = 0;
3121 curfid = 0;
3122 inodelist = [];
3123 inodetofile = {};
3124 filetoinode = {};
3125 inodetocatinode = {};
3126 if(infile=="-"):
3127 infile = BytesIO();
3128 if(hasattr(sys.stdin, "buffer")):
3129 shutil.copyfileobj(sys.stdin.buffer, infile);
3130 else:
3131 shutil.copyfileobj(sys.stdin, infile);
3132 infile.seek(0, 0);
3133 if(not infile):
3134 return False;
3135 infile.seek(0, 0);
3136 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3137 infile = download_file_from_internet_file(infile);
3138 infile.seek(0, 0);
3139 if(not infile):
3140 return False;
3141 infile.seek(0, 0);
3142 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3143 return False;
3144 elif(os.path.exists(infile) and os.path.isfile(infile)):
3145 try:
3146 if(not tarfile.TarFileCheck(infile)):
3147 return False;
3148 except AttributeError:
3149 if(not TarFileCheck(infile)):
3150 return False;
3151 try:
3152 if(hasattr(infile, "read") or hasattr(infile, "write")):
3153 tarfp = tarfile.open(fileobj=infile, mode="r");
3154 else:
3155 tarfp = tarfile.open(infile, "r");
3156 except FileNotFoundError:
3157 return False;
3158 numfiles = int(len(tarfp.getmembers()));
3159 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3160 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3161 if(re.findall(r"^[.|/]", member.name)):
3162 fname = member.name;
3163 else:
3164 fname = "./"+member.name;
3165 if(verbose):
3166 VerbosePrintOut(fname);
3167 fpremode = member.mode;
3168 ffullmode = member.mode;
3169 flinkcount = 0;
3170 ftype = 0;
3171 if(member.isreg()):
3172 ffullmode = member.mode + stat.S_IFREG;
3173 ftype = 0;
3174 elif(member.isdev()):
3175 ffullmode = member.mode;
3176 ftype = 7;
3177 elif(member.islnk()):
3178 ffullmode = member.mode + stat.S_IFREG;
3179 ftype = 1;
3180 elif(member.issym()):
3181 ffullmode = member.mode + stat.S_IFLNK;
3182 ftype = 2;
3183 elif(member.ischr()):
3184 ffullmode = member.mode + stat.S_IFCHR;
3185 ftype = 3;
3186 elif(member.isblk()):
3187 ffullmode = member.mode + stat.S_IFBLK;
3188 ftype = 4;
3189 elif(member.isdir()):
3190 ffullmode = member.mode + stat.S_IFDIR;
3191 ftype = 5;
3192 elif(member.isfifo()):
3193 ffullmode = member.mode + stat.S_IFIFO;
3194 ftype = 6;
3195 elif(member.issparse()):
3196 ffullmode = member.mode;
3197 ftype = 12;
3198 else:
3199 ffullmode = member.mode;
3200 ftype = 0;
3201 flinkname = "";
3202 fcurfid = format(int(curfid), 'x').lower();
3203 fcurinode = format(int(curfid), 'x').lower();
3204 curfid = curfid + 1;
3205 if(ftype==2):
3206 flinkname = member.linkname;
3207 fdev_minor = format(int(member.devminor), 'x').lower();
3208 fdev_major = format(int(member.devmajor), 'x').lower();
3209 frdev_minor = format(int(member.devminor), 'x').lower();
3210 frdev_major = format(int(member.devmajor), 'x').lower();
3211 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3212 fsize = format(int("0"), 'x').lower();
3213 elif(ftype==0 or ftype==7):
3214 fsize = format(int(member.size), 'x').lower();
3215 else:
3216 fsize = format(int(member.size), 'x').lower();
3217 fatime = format(int(member.mtime), 'x').lower();
3218 fmtime = format(int(member.mtime), 'x').lower();
3219 fctime = format(int(member.mtime), 'x').lower();
3220 fbtime = format(int(member.mtime), 'x').lower();
3221 fmode = format(int(ffullmode), 'x').lower();
3222 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3223 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3224 fuid = format(int(member.uid), 'x').lower();
3225 fgid = format(int(member.gid), 'x').lower();
3226 funame = member.uname;
3227 fgname = member.gname;
3228 flinkcount = format(int(flinkcount), 'x').lower();
3229 fwinattributes = format(int(0), 'x').lower();
3230 fcompression = "";
3231 fcsize = format(int(0), 'x').lower();
3232 fcontents = BytesIO();
3233 if(ftype==0 or ftype==7):
3234 with tarfp.extractfile(member) as fpc:
3235 shutil.copyfileobj(fpc, fcontents);
3236 if(not compresswholefile):
3237 fcontents.seek(0, 2);
3238 ucfsize = fcontents.tell();
3239 fcontents.seek(0, 0);
3240 if(compression=="auto"):
3241 ilsize = len(compressionlistalt);
3242 ilmin = 0;
3243 ilcsize = [];
3244 while(ilmin < ilsize):
3245 cfcontents = BytesIO();
3246 shutil.copyfileobj(fcontents, cfcontents);
3247 fcontents.seek(0, 0);
3248 cfcontents.seek(0, 0);
3249 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3250 if(cfcontents):
3251 cfcontents.seek(0, 2);
3252 ilcsize.append(cfcontents.tell());
3253 cfcontents.close();
3254 else:
3255 try:
3256 ilcsize.append(sys.maxint);
3257 except AttributeError:
3258 ilcsize.append(sys.maxsize);
3259 ilmin = ilmin + 1;
3260 ilcmin = ilcsize.index(min(ilcsize));
3261 compression = compressionlistalt[ilcmin];
3262 fcontents.seek(0, 0);
3263 cfcontents = BytesIO();
3264 shutil.copyfileobj(fcontents, cfcontents);
3265 cfcontents.seek(0, 0);
3266 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3267 cfcontents.seek(0, 2);
3268 cfsize = cfcontents.tell();
3269 if(ucfsize > cfsize):
3270 fcsize = format(int(cfsize), 'x').lower();
3271 fcompression = compression;
3272 fcontents.close();
3273 fcontents = cfcontents;
3274 if(fcompression=="none"):
3275 fcompression = "";
3276 fcontents.seek(0, 0);
3277 ftypehex = format(ftype, 'x').lower();
3278 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3279 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3280 fcontents.close();
3281 if(numfiles>0):
3282 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3283 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3284 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3285 try:
3286 catfp.flush();
3287 os.fsync(catfp.fileno());
3288 except io.UnsupportedOperation:
3289 pass;
3290 except AttributeError:
3291 pass;
3292 except OSError as e:
3293 pass;
3294 if(outfile=="-"):
3295 catfp.seek(0, 0);
3296 if(hasattr(sys.stdout, "buffer")):
3297 shutil.copyfileobj(catfp, sys.stdout.buffer);
3298 else:
3299 shutil.copyfileobj(catfp, sys.stdout);
3300 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3301 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3302 catfp.seek(0, 0);
3303 upload_file_to_internet_file(catfp, outfile);
3304 if(returnfp):
3305 catfp.seek(0, 0);
3306 return catfp;
3307 else:
3308 catfp.close();
3309 return True;
3310
3311 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3312
3313 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3314 formatspecs = FormatSpecsListToDict(formatspecs);
3315 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3316 outfile = RemoveWindowsPath(outfile);
3317 checksumtype = checksumtype.lower();
3318 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3319 checksumtype="crc32";
3320 if(checksumtype=="none"):
3321 checksumtype = "";
3322 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3323 compression = "auto";
3324 if(compression not in compressionlist and compression is None):
3325 compression = "auto";
3326 if(verbose):
3327 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3328 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3329 if(os.path.exists(outfile)):
3330 try:
3331 os.unlink(outfile);
3332 except OSError as e:
3333 pass;
3334 if(outfile=="-"):
3335 verbose = False;
3336 catfp = BytesIO();
3337 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3338 catfp = outfile;
3339 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3340 catfp = BytesIO();
3341 else:
3342 fbasename = os.path.splitext(outfile)[0];
3343 fextname = os.path.splitext(outfile)[1];
3344 if(not compresswholefile and fextname in outextlistwd):
3345 compresswholefile = True;
3346 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3347 catver = formatspecs['format_ver'];
3348 fileheaderver = str(int(catver.replace(".", "")));
3349 curinode = 0;
3350 curfid = 0;
3351 inodelist = [];
3352 inodetofile = {};
3353 filetoinode = {};
3354 inodetocatinode = {};
3355 if(infile=="-"):
3356 infile = BytesIO();
3357 if(hasattr(sys.stdin, "buffer")):
3358 shutil.copyfileobj(sys.stdin.buffer, infile);
3359 else:
3360 shutil.copyfileobj(sys.stdin, infile);
3361 infile.seek(0, 0);
3362 if(not infile):
3363 return False;
3364 infile.seek(0, 0);
3365 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3366 infile = download_file_from_internet_file(infile);
3367 infile.seek(0, 0);
3368 if(not infile):
3369 return False;
3370 infile.seek(0, 0);
3371 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3372 return False;
3373 if(not zipfile.is_zipfile(infile)):
3374 return False;
3375 try:
3376 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3377 except FileNotFoundError:
3378 return False;
3379 ziptest = zipfp.testzip();
3380 if(ziptest):
3381 VerbosePrintOut("Bad file found!");
3382 numfiles = int(len(zipfp.infolist()));
3383 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3384 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3385 if(re.findall(r"^[.|/]", member.filename)):
3386 fname = member.filename;
3387 else:
3388 fname = "./"+member.filename;
3389 zipinfo = zipfp.getinfo(member.filename);
3390 if(verbose):
3391 VerbosePrintOut(fname);
3392 if(not member.is_dir()):
3393 fpremode = int(stat.S_IFREG + 438);
3394 elif(member.is_dir()):
3395 fpremode = int(stat.S_IFDIR + 511);
3396 flinkcount = 0;
3397 ftype = 0;
3398 if(not member.is_dir()):
3399 ftype = 0;
3400 elif(member.is_dir()):
3401 ftype = 5;
3402 flinkname = "";
3403 fcurfid = format(int(curfid), 'x').lower();
3404 fcurinode = format(int(curfid), 'x').lower();
3405 curfid = curfid + 1;
3406 fdev_minor = format(int(0), 'x').lower();
3407 fdev_major = format(int(0), 'x').lower();
3408 frdev_minor = format(int(0), 'x').lower();
3409 frdev_major = format(int(0), 'x').lower();
3410 if(ftype==5):
3411 fsize = format(int("0"), 'x').lower();
3412 elif(ftype==0):
3413 fsize = format(int(member.file_size), 'x').lower();
3414 else:
3415 fsize = format(int(member.file_size), 'x').lower();
3416 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3417 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3418 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3419 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3420 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3421 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3422 if(not member.is_dir()):
3423 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3424 fchmode = stat.S_IMODE(int(stat.S_IFREG + 438));
3425 ftypemod = stat.S_IFMT(int(stat.S_IFREG + 438));
3426 elif(member.is_dir()):
3427 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3428 fchmode = stat.S_IMODE(int(stat.S_IFDIR + 511));
3429 ftypemod = stat.S_IFMT(int(stat.S_IFDIR + 511));
3430 elif(zipinfo.create_system==3):
3431 fwinattributes = format(int(0), 'x').lower();
3432 fmode = format(int(zipinfo.external_attr), 'x').lower();
3433 fchmode = stat.S_IMODE(int(zipinfo.external_attr));
3434 ftypemod = stat.S_IFMT(int(zipinfo.external_attr));
3435 else:
3436 fwinattributes = format(int(0), 'x').lower();
3437 if(not member.is_dir()):
3438 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3439 fchmode = stat.S_IMODE(fmode);
3440 ftypemod = stat.S_IFMT(fmode);
3441 elif(member.is_dir()):
3442 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3443 fchmode = stat.S_IMODE(fmode);
3444 ftypemod = stat.S_IFMT(fmode);
3445 fcompression = "";
3446 fcsize = format(int(0), 'x').lower();
3447 try:
3448 fuid = format(int(os.getuid()), 'x').lower();
3449 except AttributeError:
3450 fuid = format(int(0), 'x').lower();
3451 except KeyError:
3452 fuid = format(int(0), 'x').lower();
3453 try:
3454 fgid = format(int(os.getgid()), 'x').lower();
3455 except AttributeError:
3456 fgid = format(int(0), 'x').lower();
3457 except KeyError:
3458 fgid = format(int(0), 'x').lower();
3459 try:
3460 import pwd;
3461 try:
3462 userinfo = pwd.getpwuid(os.getuid());
3463 funame = userinfo.pw_name;
3464 except KeyError:
3465 funame = "";
3466 except AttributeError:
3467 funame = "";
3468 except ImportError:
3469 funame = "";
3470 fgname = "";
3471 try:
3472 import grp;
3473 try:
3474 groupinfo = grp.getgrgid(os.getgid());
3475 fgname = groupinfo.gr_name;
3476 except KeyError:
3477 fgname = "";
3478 except AttributeError:
3479 fgname = "";
3480 except ImportError:
3481 fgname = "";
3482 fcontents = BytesIO();
3483 if(ftype==0):
3484 fcontents.write(zipfp.read(member.filename));
3485 if(not compresswholefile):
3486 fcontents.seek(0, 2);
3487 ucfsize = fcontents.tell();
3488 fcontents.seek(0, 0);
3489 if(compression=="auto"):
3490 ilsize = len(compressionlistalt);
3491 ilmin = 0;
3492 ilcsize = [];
3493 while(ilmin < ilsize):
3494 cfcontents = BytesIO();
3495 shutil.copyfileobj(fcontents, cfcontents);
3496 fcontents.seek(0, 0);
3497 cfcontents.seek(0, 0);
3498 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3499 cfcontents.seek(0, 2);
3500 ilcsize.append(cfcontents.tell());
3501 cfcontents.close();
3502 ilmin = ilmin + 1;
3503 ilcmin = ilcsize.index(min(ilcsize));
3504 compression = compressionlistalt[ilcmin];
3505 fcontents.seek(0, 0);
3506 cfcontents = BytesIO();
3507 shutil.copyfileobj(fcontents, cfcontents);
3508 cfcontents.seek(0, 0);
3509 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3510 cfcontents.seek(0, 2);
3511 cfsize = cfcontents.tell();
3512 if(ucfsize > cfsize):
3513 fcsize = format(int(cfsize), 'x').lower();
3514 fcompression = compression;
3515 fcontents.close();
3516 fcontents = cfcontents;
3517 if(fcompression=="none"):
3518 fcompression = "";
3519 fcontents.seek(0, 0);
3520 ftypehex = format(ftype, 'x').lower();
3521 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3522 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3523 fcontents.close();
3524 if(numfiles>0):
3525 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3526 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3527 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3528 try:
3529 catfp.flush();
3530 os.fsync(catfp.fileno());
3531 except io.UnsupportedOperation:
3532 pass;
3533 except AttributeError:
3534 pass;
3535 except OSError as e:
3536 pass;
3537 if(outfile=="-"):
3538 catfp.seek(0, 0);
3539 if(hasattr(sys.stdout, "buffer")):
3540 shutil.copyfileobj(catfp, sys.stdout.buffer);
3541 else:
3542 shutil.copyfileobj(catfp, sys.stdout);
3543 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3544 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3545 catfp.seek(0, 0);
3546 upload_file_to_internet_file(catfp, outfile);
3547 if(returnfp):
3548 catfp.seek(0, 0);
3549 return catfp;
3550 else:
3551 catfp.close();
3552 return True;
3553
3554 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3555
3556 if(not rarfile_support):
3557 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3558 return False
3559
3560 if(rarfile_support):
3561 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3562 formatspecs = FormatSpecsListToDict(formatspecs);
3563 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3564 outfile = RemoveWindowsPath(outfile);
3565 checksumtype = checksumtype.lower();
3566 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3567 checksumtype="crc32";
3568 if(checksumtype=="none"):
3569 checksumtype = "";
3570 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3571 compression = "auto";
3572 if(compression not in compressionlist and compression is None):
3573 compression = "auto";
3574 if(verbose):
3575 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3576 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3577 if(os.path.exists(outfile)):
3578 try:
3579 os.unlink(outfile);
3580 except OSError as e:
3581 pass;
3582 if(outfile=="-"):
3583 verbose = False;
3584 catfp = BytesIO();
3585 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3586 catfp = outfile;
3587 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3588 catfp = BytesIO();
3589 else:
3590 fbasename = os.path.splitext(outfile)[0];
3591 fextname = os.path.splitext(outfile)[1];
3592 if(not compresswholefile and fextname in outextlistwd):
3593 compresswholefile = True;
3594 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3595 catver = formatspecs['format_ver'];
3596 fileheaderver = str(int(catver.replace(".", "")));
3597 curinode = 0;
3598 curfid = 0;
3599 inodelist = [];
3600 inodetofile = {};
3601 filetoinode = {};
3602 inodetocatinode = {};
3603 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3604 return False;
3605 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3606 return False;
3607 rarfp = rarfile.RarFile(infile, "r");
3608 rartest = rarfp.testrar();
3609 if(rartest):
3610 VerbosePrintOut("Bad file found!");
3611 numfiles = int(len(rarfp.infolist()));
3612 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3613 try:
3614 catfp.flush();
3615 os.fsync(catfp.fileno());
3616 except io.UnsupportedOperation:
3617 pass;
3618 except AttributeError:
3619 pass;
3620 except OSError as e:
3621 pass;
3622 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3623 is_unix = False;
3624 is_windows = False;
3625 if(member.host_os==rarfile.RAR_OS_UNIX):
3626 is_windows = False;
3627 try:
3628 member.external_attr
3629 is_unix = True;
3630 except AttributeError:
3631 is_unix = False;
3632 elif(member.host_os==rarfile.RAR_OS_WIN32):
3633 is_unix = False;
3634 try:
3635 member.external_attr
3636 is_windows = True;
3637 except AttributeError:
3638 is_windows = False;
3639 else:
3640 is_unix = False;
3641 is_windows = False;
3642 if(re.findall(r"^[.|/]", member.filename)):
3643 fname = member.filename;
3644 else:
3645 fname = "./"+member.filename;
3646 rarinfo = rarfp.getinfo(member.filename);
3647 if(verbose):
3648 VerbosePrintOut(fname);
3649 if(is_unix and member.external_attr !=0):
3650 fpremode = int(member.external_attr);
3651 elif(member.is_file()):
3652 fpremode = int(stat.S_IFREG + 438);
3653 elif(member.is_symlink()):
3654 fpremode = int(stat.S_IFLNK + 438);
3655 elif(member.is_dir()):
3656 fpremode = int(stat.S_IFDIR + 511);
3657 if(is_windows and member.external_attr !=0):
3658 fwinattributes = format(int(member.external_attr), 'x').lower();
3659 else:
3660 fwinattributes = format(int(0), 'x').lower();
3661 fcompression = "";
3662 fcsize = format(int(0), 'x').lower();
3663 flinkcount = 0;
3664 ftype = 0;
3665 if(member.is_file()):
3666 ftype = 0;
3667 elif(member.is_symlink()):
3668 ftype = 2;
3669 elif(member.is_dir()):
3670 ftype = 5;
3671 flinkname = "";
3672 if(ftype==2):
3673 flinkname = rarfp.read(member.filename).decode("UTF-8");
3674 fcurfid = format(int(curfid), 'x').lower();
3675 fcurinode = format(int(curfid), 'x').lower();
3676 curfid = curfid + 1;
3677 fdev_minor = format(int(0), 'x').lower();
3678 fdev_major = format(int(0), 'x').lower();
3679 frdev_minor = format(int(0), 'x').lower();
3680 frdev_major = format(int(0), 'x').lower();
3681 if(ftype==5):
3682 fsize = format(int("0"), 'x').lower();
3683 elif(ftype==0):
3684 fsize = format(int(member.file_size), 'x').lower();
3685 else:
3686 fsize = format(int(member.file_size), 'x').lower();
3687 try:
3688 if(member.atime):
3689 fatime = format(int(member.atime.timestamp()), 'x').lower();
3690 else:
3691 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3692 except AttributeError:
3693 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3694 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3695 try:
3696 if(member.ctime):
3697 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3698 else:
3699 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3700 except AttributeError:
3701 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3702 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3703 if(is_unix and member.external_attr !=0):
3704 fmode = format(int(member.external_attr), 'x').lower();
3705 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3706 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3707 elif(member.is_file()):
3708 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3709 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3710 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3711 elif(member.is_symlink()):
3712 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3713 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3714 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3715 elif(member.is_dir()):
3716 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3717 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3718 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3719 try:
3720 fuid = format(int(os.getuid()), 'x').lower();
3721 except AttributeError:
3722 fuid = format(int(0), 'x').lower();
3723 except KeyError:
3724 fuid = format(int(0), 'x').lower();
3725 try:
3726 fgid = format(int(os.getgid()), 'x').lower();
3727 except AttributeError:
3728 fgid = format(int(0), 'x').lower();
3729 except KeyError:
3730 fgid = format(int(0), 'x').lower();
3731 try:
3732 import pwd;
3733 try:
3734 userinfo = pwd.getpwuid(os.getuid());
3735 funame = userinfo.pw_name;
3736 except KeyError:
3737 funame = "";
3738 except AttributeError:
3739 funame = "";
3740 except ImportError:
3741 funame = "";
3742 fgname = "";
3743 try:
3744 import grp;
3745 try:
3746 groupinfo = grp.getgrgid(os.getgid());
3747 fgname = groupinfo.gr_name;
3748 except KeyError:
3749 fgname = "";
3750 except AttributeError:
3751 fgname = "";
3752 except ImportError:
3753 fgname = "";
3754 fcontents = BytesIO();
3755 if(ftype==0):
3756 fcontents.write(rarfp.read(member.filename));
3757 if(not compresswholefile):
3758 fcontents.seek(0, 2);
3759 ucfsize = fcontents.tell();
3760 fcontents.seek(0, 0);
3761 if(compression=="auto"):
3762 ilsize = len(compressionlistalt);
3763 ilmin = 0;
3764 ilcsize = [];
3765 while(ilmin < ilsize):
3766 cfcontents = BytesIO();
3767 shutil.copyfileobj(fcontents, cfcontents);
3768 fcontents.seek(0, 0);
3769 cfcontents.seek(0, 0);
3770 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3771 if(cfcontents):
3772 cfcontents.seek(0, 2);
3773 ilcsize.append(cfcontents.tell());
3774 cfcontents.close();
3775 else:
3776 try:
3777 ilcsize.append(sys.maxint);
3778 except AttributeError:
3779 ilcsize.append(sys.maxsize);
3780 ilmin = ilmin + 1;
3781 ilcmin = ilcsize.index(min(ilcsize));
3782 compression = compressionlistalt[ilcmin];
3783 fcontents.seek(0, 0);
3784 cfcontents = BytesIO();
3785 shutil.copyfileobj(fcontents, cfcontents);
3786 cfcontents.seek(0, 0);
3787 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3788 cfcontents.seek(0, 2);
3789 cfsize = cfcontents.tell();
3790 if(ucfsize > cfsize):
3791 fcsize = format(int(cfsize), 'x').lower();
3792 fcompression = compression;
3793 fcontents.close();
3794 fcontents = cfcontents;
3795 if(fcompression=="none"):
3796 fcompression = "";
3797 fcontents.seek(0, 0);
3798 ftypehex = format(ftype, 'x').lower();
3799 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3800 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3801 fcontents.close();
3802 if(numfiles>0):
3803 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3804 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3805 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3806 try:
3807 catfp.flush();
3808 os.fsync(catfp.fileno());
3809 except io.UnsupportedOperation:
3810 pass;
3811 except AttributeError:
3812 pass;
3813 except OSError as e:
3814 pass;
3815 if(outfile=="-"):
3816 catfp.seek(0, 0)
3817 if(hasattr(sys.stdout, "buffer")):
3818 shutil.copyfileobj(catfp, sys.stdout.buffer);
3819 else:
3820 shutil.copyfileobj(catfp, sys.stdout);
3821 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3822 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3823 catfp.seek(0, 0);
3824 upload_file_to_internet_file(catfp, outfile);
3825 if(returnfp):
3826 catfp.seek(0, 0)
3827 return catfp
3828 else:
3829 catfp.close()
3830 return True;
3831
3832 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3833
3834 if(not py7zr_support):
3835 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3836 return False
3837
3838 if(py7zr_support):
3839 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3840 formatspecs = FormatSpecsListToDict(formatspecs);
3841 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3842 outfile = RemoveWindowsPath(outfile);
3843 checksumtype = checksumtype.lower();
3844 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3845 checksumtype="crc32";
3846 if(checksumtype=="none"):
3847 checksumtype = "";
3848 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3849 compression = "auto";
3850 if(compression not in compressionlist and compression is None):
3851 compression = "auto";
3852 if(verbose):
3853 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3854 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3855 if(os.path.exists(outfile)):
3856 try:
3857 os.unlink(outfile);
3858 except OSError as e:
3859 pass;
3860 if(outfile=="-"):
3861 verbose = False;
3862 catfp = BytesIO();
3863 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3864 catfp = outfile;
3865 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3866 catfp = BytesIO();
3867 else:
3868 fbasename = os.path.splitext(outfile)[0];
3869 fextname = os.path.splitext(outfile)[1];
3870 if(not compresswholefile and fextname in outextlistwd):
3871 compresswholefile = True;
3872 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3873 catver = formatspecs['format_ver'];
3874 fileheaderver = str(int(catver.replace(".", "")));
3875 curinode = 0;
3876 curfid = 0;
3877 inodelist = [];
3878 inodetofile = {};
3879 filetoinode = {};
3880 inodetocatinode = {};
3881 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3882 return False;
3883 szpfp = py7zr.SevenZipFile(infile, mode="r");
3884 file_content = szpfp.readall();
3885 #sztest = szpfp.testzip();
3886 sztestalt = szpfp.test();
3887 if(sztestalt):
3888 VerbosePrintOut("Bad file found!");
3889 numfiles = int(len(szpfp.list()));
3890 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3891 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3892 if(re.findall(r"^[.|/]", member.filename)):
3893 fname = member.filename;
3894 else:
3895 fname = "./"+member.filename;
3896 if(verbose):
3897 VerbosePrintOut(fname);
3898 if(not member.is_directory):
3899 fpremode = int(stat.S_IFREG + 438);
3900 elif(member.is_directory):
3901 fpremode = int(stat.S_IFDIR + 511);
3902 fwinattributes = format(int(0), 'x').lower();
3903 fcompression = "";
3904 fcsize = format(int(0), 'x').lower();
3905 flinkcount = 0;
3906 ftype = 0;
3907 if(member.is_directory):
3908 ftype = 5;
3909 else:
3910 ftype = 0;
3911 flinkname = "";
3912 fcurfid = format(int(curfid), 'x').lower();
3913 fcurinode = format(int(curfid), 'x').lower();
3914 curfid = curfid + 1;
3915 fdev_minor = format(int(0), 'x').lower();
3916 fdev_major = format(int(0), 'x').lower();
3917 frdev_minor = format(int(0), 'x').lower();
3918 frdev_major = format(int(0), 'x').lower();
3919 if(ftype==5):
3920 fsize = format(int("0"), 'x').lower();
3921 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3922 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3923 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3924 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3925 if(member.is_directory):
3926 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3927 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3928 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3929 else:
3930 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3931 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3932 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3933 try:
3934 fuid = format(int(os.getuid()), 'x').lower();
3935 except AttributeError:
3936 fuid = format(int(0), 'x').lower();
3937 except KeyError:
3938 fuid = format(int(0), 'x').lower();
3939 try:
3940 fgid = format(int(os.getgid()), 'x').lower();
3941 except AttributeError:
3942 fgid = format(int(0), 'x').lower();
3943 except KeyError:
3944 fgid = format(int(0), 'x').lower();
3945 try:
3946 import pwd;
3947 try:
3948 userinfo = pwd.getpwuid(os.getuid());
3949 funame = userinfo.pw_name;
3950 except KeyError:
3951 funame = "";
3952 except AttributeError:
3953 funame = "";
3954 except ImportError:
3955 funame = "";
3956 fgname = "";
3957 try:
3958 import grp;
3959 try:
3960 groupinfo = grp.getgrgid(os.getgid());
3961 fgname = groupinfo.gr_name;
3962 except KeyError:
3963 fgname = "";
3964 except AttributeError:
3965 fgname = "";
3966 except ImportError:
3967 fgname = "";
3968 fcontents = BytesIO();
3969 if(ftype==0):
3970 fcontents.write(file_content[member.filename].read());
3971 fsize = format(fcontents.tell(), 'x').lower();
3972 file_content[member.filename].close();
3973 if(not compresswholefile):
3974 fcontents.seek(0, 2);
3975 ucfsize = fcontents.tell();
3976 fcontents.seek(0, 0);
3977 if(compression=="auto"):
3978 ilsize = len(compressionlistalt);
3979 ilmin = 0;
3980 ilcsize = [];
3981 while(ilmin < ilsize):
3982 cfcontents = BytesIO();
3983 shutil.copyfileobj(fcontents, cfcontents);
3984 fcontents.seek(0, 0);
3985 cfcontents.seek(0, 0);
3986 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3987 if(cfcontents):
3988 cfcontents.seek(0, 2);
3989 ilcsize.append(cfcontents.tell());
3990 cfcontents.close();
3991 else:
3992 try:
3993 ilcsize.append(sys.maxint);
3994 except AttributeError:
3995 ilcsize.append(sys.maxsize);
3996 ilmin = ilmin + 1;
3997 ilcmin = ilcsize.index(min(ilcsize));
3998 compression = compressionlistalt[ilcmin];
3999 fcontents.seek(0, 0);
4000 cfcontents = BytesIO();
4001 shutil.copyfileobj(fcontents, cfcontents);
4002 cfcontents.seek(0, 0);
4003 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4004 cfcontents.seek(0, 2);
4005 cfsize = cfcontents.tell();
4006 if(ucfsize > cfsize):
4007 fcsize = format(int(cfsize), 'x').lower();
4008 fcompression = compression;
4009 fcontents.close();
4010 fcontents = cfcontents;
4011 if(fcompression=="none"):
4012 fcompression = "";
4013 fcontents.seek(0, 0);
4014 ftypehex = format(ftype, 'x').lower();
4015 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4016 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4017 fcontents.close();
4018 if(numfiles>0):
4019 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4020 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4021 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4022 try:
4023 catfp.flush();
4024 os.fsync(catfp.fileno());
4025 except io.UnsupportedOperation:
4026 pass;
4027 except AttributeError:
4028 pass;
4029 except OSError as e:
4030 pass;
4031 if(outfile=="-"):
4032 catfp.seek(0, 0);
4033 if(hasattr(sys.stdout, "buffer")):
4034 shutil.copyfileobj(catfp, sys.stdout.buffer);
4035 else:
4036 shutil.copyfileobj(catfp, sys.stdout);
4037 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4038 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4039 catfp.seek(0, 0);
4040 upload_file_to_internet_file(catfp, outfile);
4041 if(returnfp):
4042 catfp.seek(0, 0)
4043 return catfp
4044 else:
4045 catfp.close()
4046 return True;
4047
4048 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4049
4050 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4051 formatspecs = FormatSpecsListToDict(formatspecs);
4052 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4053 if(verbose):
4054 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4055 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4056 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4057 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4058 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4059 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4060 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4061 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4062 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4063 elif(checkcompressfile=="catfile"):
4064 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4065 else:
4066 return False;
4067 return False;
4068
4069 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4070
4071 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4072 formatspecs = FormatSpecsListToDict(formatspecs);
4073 if(hasattr(infile, "read") or hasattr(infile, "write")):
4074 catfp = infile;
4075 catfp.seek(0, 0);
4076 catfp = UncompressArchiveFile(catfp, formatspecs);
4077 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4078 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4079 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4080 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4081 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4082 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4083 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4084 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4085 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4086 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4087 return False;
4088 if(not catfp):
4089 return False;
4090 catfp.seek(0, 0);
4091 elif(infile=="-"):
4092 catfp = BytesIO();
4093 if(hasattr(sys.stdin, "buffer")):
4094 shutil.copyfileobj(sys.stdin.buffer, catfp);
4095 else:
4096 shutil.copyfileobj(sys.stdin, catfp);
4097 catfp.seek(0, 0);
4098 catfp = UncompressArchiveFile(catfp, formatspecs);
4099 if(not catfp):
4100 return False;
4101 catfp.seek(0, 0);
4102 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4103 catfp = download_file_from_internet_file(infile);
4104 catfp.seek(0, 0);
4105 catfp = UncompressArchiveFile(catfp, formatspecs);
4106 if(not catfp):
4107 return False;
4108 catfp.seek(0, 0);
4109 else:
4110 infile = RemoveWindowsPath(infile);
4111 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4112 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4113 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4114 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4115 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4116 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4117 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4118 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4119 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4120 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4121 return False;
4122 compresscheck = CheckCompressionType(infile, formatspecs, True);
4123 if(not compresscheck):
4124 fextname = os.path.splitext(infile)[1];
4125 if(fextname==".gz"):
4126 compresscheck = "gzip";
4127 elif(fextname==".bz2"):
4128 compresscheck = "bzip2";
4129 elif(fextname==".zst"):
4130 compresscheck = "zstd";
4131 elif(fextname==".lz4" or fextname==".clz4"):
4132 compresscheck = "lz4";
4133 elif(fextname==".lzo" or fextname==".lzop"):
4134 compresscheck = "lzo";
4135 elif(fextname==".lzma" or fextname==".xz"):
4136 compresscheck = "lzma";
4137 else:
4138 return False;
4139 if(not compresscheck):
4140 return False;
4141 catfp = UncompressFile(infile, formatspecs, "rb");
4142 '''
4143 try:
4144 catfp.seek(0, 2);
4145 except OSError:
4146 SeekToEndOfFile(catfp);
4147 except ValueError:
4148 SeekToEndOfFile(catfp);
4149 CatSize = catfp.tell();
4150 CatSizeEnd = CatSize;
4151 '''
4152 try:
4153 catfp.seek(0, 0);
4154 except OSError:
4155 return False;
4156 except ValueError:
4157 return False;
4158 curloc = catfp.tell();
4159 if(curloc>0):
4160 catfp.seek(0, 0);
4161 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4162 if(curloc>0):
4163 catfp.seek(curloc, 0);
4164 catstring = catheader[0];
4165 catversion = re.findall(r"([\d]+)", catstring);
4166 fprenumfiles = catheader[1];
4167 fnumfiles = int(fprenumfiles, 16);
4168 fprechecksumtype = catheader[2];
4169 fprechecksum = catheader[3];
4170 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4171 fnumfileshex = format(int(fnumfiles), 'x').lower();
4172 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4173 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4174 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4175 fheadtell = len(fileheader);
4176 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4177 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4178 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4179 return False;
4180 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4181 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4182 if(seekto>=fnumfiles):
4183 seekto = fnumfiles - 1;
4184 if(seekto<0):
4185 seekto = 0;
4186 if(seekto>=0):
4187 il = -1;
4188 while(il < seekto):
4189 prefhstart = catfp.tell();
4190 if(formatspecs['new_style']):
4191 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4192 else:
4193 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4194 if(len(preheaderdata)==0):
4195 break;
4196 prefheadsize = int(preheaderdata[0], 16);
4197 prefnumfields = int(preheaderdata[1], 16);
4198 preftype = int(preheaderdata[2], 16);
4199 if(re.findall(r"^[.|/]", preheaderdata[3])):
4200 prefname = preheaderdata[3];
4201 else:
4202 prefname = "./"+preheaderdata[3];
4203 prefbasedir = os.path.dirname(prefname);
4204 preflinkname = preheaderdata[4];
4205 prefsize = int(preheaderdata[5], 16);
4206 prefatime = int(preheaderdata[6], 16);
4207 prefmtime = int(preheaderdata[7], 16);
4208 prefctime = int(preheaderdata[8], 16);
4209 prefbtime = int(preheaderdata[9], 16);
4210 prefmode = int(preheaderdata[10], 16);
4211 prefchmode = stat.S_IMODE(prefmode);
4212 preftypemod = stat.S_IFMT(prefmode);
4213 prefwinattributes = int(preheaderdata[11], 16);
4214 prefcompression = preheaderdata[12];
4215 prefcsize = int(preheaderdata[13], 16);
4216 prefuid = int(preheaderdata[14], 16);
4217 prefuname = preheaderdata[15];
4218 prefgid = int(preheaderdata[16], 16);
4219 prefgname = preheaderdata[17];
4220 fid = int(preheaderdata[18], 16);
4221 finode = int(preheaderdata[19], 16);
4222 flinkcount = int(preheaderdata[20], 16);
4223 prefdev_minor = int(preheaderdata[21], 16);
4224 prefdev_major = int(preheaderdata[22], 16);
4225 prefrdev_minor = int(preheaderdata[23], 16);
4226 prefrdev_major = int(preheaderdata[24], 16);
4227 prefseeknextfile = preheaderdata[25];
4228 prefextrasize = int(preheaderdata[26], 16);
4229 prefextrafields = int(preheaderdata[27], 16);
4230 extrafieldslist = [];
4231 extrastart = 28;
4232 extraend = extrastart + prefextrafields;
4233 extrafieldslist = [];
4234 if(extrastart<extraend):
4235 extrafieldslist.append(preheaderdata[extrastart]);
4236 extrastart = extrastart + 1;
4237 prefcs = preheaderdata[-2].lower();
4238 prenewfcs = preheaderdata[-1].lower();
4239 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4240 if(prefcs!=prenewfcs and not skipchecksum):
4241 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4242 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4243 return False;
4244 valid_archive = False;
4245 invalid_archive = True;
4246 prefhend = catfp.tell() - 1;
4247 prefcontentstart = catfp.tell();
4248 prefcontents = "";
4249 pyhascontents = False;
4250 if(prefsize>0):
4251 if(prefcompression):
4252 prefcontents = catfp.read(prefsize);
4253 else:
4254 prefcontents = catfp.read(prefcsize);
4255 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4256 pyhascontents = True;
4257 if(prefccs!=prenewfccs and not skipchecksum):
4258 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4259 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4260 return False;
4261 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4262 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4263 if(abs(fseeknextasnum)==0):
4264 pass;
4265 catfp.seek(fseeknextasnum, 1);
4266 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4267 fseeknextasnum = int(prefseeknextfile);
4268 if(abs(fseeknextasnum)==0):
4269 pass;
4270 catfp.seek(fseeknextasnum, 1);
4271 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4272 fseeknextasnum = int(prefseeknextfile);
4273 if(abs(fseeknextasnum)==0):
4274 pass;
4275 catfp.seek(fseeknextasnum, 0);
4276 else:
4277 return False;
4278 il = il + 1;
4279 catfp.seek(seekstart, 0);
4280 fileidnum = il;
4281 catfheadsize = int(preheaderdata[0], 16);
4282 catfnumfields = int(preheaderdata[1], 16);
4283 catftype = int(preheaderdata[2], 16);
4284 if(re.findall(r"^[.|/]", preheaderdata[3])):
4285 catfname = preheaderdata[3];
4286 else:
4287 catfname = "./"+preheaderdata[3];
4288 catflinkname = preheaderdata[4];
4289 catfsize = int(preheaderdata[5], 16);
4290 catfbasedir = os.path.dirname(catfname);
4291 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4292 if(returnfp):
4293 catlist.update({'catfp': catfp});
4294 else:
4295 catfp.close();
4296 return catlist;
4297
4298 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4299
4300 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4301 formatspecs = FormatSpecsListToDict(formatspecs);
4302 if(hasattr(infile, "read") or hasattr(infile, "write")):
4303 catfp = infile;
4304 catfp.seek(0, 0);
4305 catfp = UncompressArchiveFile(catfp, formatspecs);
4306 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4307 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4308 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4309 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4310 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4311 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4312 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4313 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4314 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4315 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4316 return False;
4317 if(not catfp):
4318 return False;
4319 catfp.seek(0, 0);
4320 elif(infile=="-"):
4321 catfp = BytesIO();
4322 if(hasattr(sys.stdin, "buffer")):
4323 shutil.copyfileobj(sys.stdin.buffer, catfp);
4324 else:
4325 shutil.copyfileobj(sys.stdin, catfp);
4326 catfp.seek(0, 0);
4327 catfp = UncompressArchiveFile(catfp, formatspecs);
4328 if(not catfp):
4329 return False;
4330 catfp.seek(0, 0);
4331 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4332 catfp = download_file_from_internet_file(infile);
4333 catfp = UncompressArchiveFile(catfp, formatspecs);
4334 catfp.seek(0, 0);
4335 if(not catfp):
4336 return False;
4337 catfp.seek(0, 0);
4338 else:
4339 infile = RemoveWindowsPath(infile);
4340 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4341 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4342 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4343 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4344 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4345 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4346 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4347 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4348 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4349 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4350 return False;
4351 compresscheck = CheckCompressionType(infile, formatspecs, True);
4352 if(not compresscheck):
4353 fextname = os.path.splitext(infile)[1];
4354 if(fextname==".gz"):
4355 compresscheck = "gzip";
4356 elif(fextname==".bz2"):
4357 compresscheck = "bzip2";
4358 elif(fextname==".zst"):
4359 compresscheck = "zstd";
4360 elif(fextname==".lz4" or fextname==".clz4"):
4361 compresscheck = "lz4";
4362 elif(fextname==".lzo" or fextname==".lzop"):
4363 compresscheck = "lzo";
4364 elif(fextname==".lzma" or fextname==".xz"):
4365 compresscheck = "lzma";
4366 else:
4367 return False;
4368 if(not compresscheck):
4369 return False;
4370 catfp = UncompressFile(infile, formatspecs, "rb");
4371 '''
4372 try:
4373 catfp.seek(0, 2);
4374 except OSError:
4375 SeekToEndOfFile(catfp);
4376 except ValueError:
4377 SeekToEndOfFile(catfp);
4378 CatSize = catfp.tell();
4379 CatSizeEnd = CatSize;
4380 '''
4381 try:
4382 catfp.seek(0, 0);
4383 except OSError:
4384 return False;
4385 except ValueError:
4386 return False;
4387 curloc = catfp.tell();
4388 if(curloc>0):
4389 catfp.seek(0, 0);
4390 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4391 if(curloc>0):
4392 catfp.seek(curloc, 0);
4393 catstring = catheader[0];
4394 catversion = re.findall(r"([\d]+)", catstring);
4395 fprenumfiles = catheader[1];
4396 fnumfiles = int(fprenumfiles, 16);
4397 fprechecksumtype = catheader[2];
4398 fprechecksum = catheader[3];
4399 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4400 fnumfileshex = format(int(fnumfiles), 'x').lower();
4401 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4402 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4403 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4404 fheadtell = len(fileheader);
4405 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4406 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4407 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4408 return False;
4409 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4410 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4411 seekto = fnumfiles - 1
4412 filefound = False;
4413 if(seekto>=0):
4414 il = -1;
4415 while(il < seekto):
4416 prefhstart = catfp.tell();
4417 if(formatspecs['new_style']):
4418 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4419 else:
4420 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4421 if(len(preheaderdata)==0):
4422 break;
4423 prefheadsize = int(preheaderdata[0], 16);
4424 prefnumfields = int(preheaderdata[1], 16);
4425 preftype = int(preheaderdata[2], 16);
4426 if(re.findall(r"^[.|/]", preheaderdata[3])):
4427 prefname = preheaderdata[3];
4428 else:
4429 prefname = "./"+preheaderdata[3];
4430 prefbasedir = os.path.dirname(prefname);
4431 preflinkname = preheaderdata[4];
4432 prefsize = int(preheaderdata[5], 16);
4433 prefatime = int(preheaderdata[6], 16);
4434 prefmtime = int(preheaderdata[7], 16);
4435 prefctime = int(preheaderdata[8], 16);
4436 prefbtime = int(preheaderdata[9], 16);
4437 prefmode = int(preheaderdata[10], 16);
4438 prefchmode = stat.S_IMODE(prefmode);
4439 preftypemod = stat.S_IFMT(prefmode);
4440 prefwinattributes = int(preheaderdata[11], 16);
4441 prefcompression = preheaderdata[12];
4442 prefcsize = int(preheaderdata[13], 16);
4443 prefuid = int(preheaderdata[14], 16);
4444 prefuname = preheaderdata[15];
4445 prefgid = int(preheaderdata[16], 16);
4446 prefgname = preheaderdata[17];
4447 fid = int(preheaderdata[18], 16);
4448 finode = int(preheaderdata[19], 16);
4449 flinkcount = int(preheaderdata[20], 16);
4450 prefdev_minor = int(preheaderdata[21], 16);
4451 prefdev_major = int(preheaderdata[22], 16);
4452 prefrdev_minor = int(preheaderdata[23], 16);
4453 prefrdev_major = int(preheaderdata[24], 16);
4454 prefseeknextfile = preheaderdata[25];
4455 prefextrasize = int(preheaderdata[26], 16);
4456 prefextrafields = int(preheaderdata[27], 16);
4457 extrafieldslist = [];
4458 extrastart = 28;
4459 extraend = extrastart + prefextrafields;
4460 extrafieldslist = [];
4461 if(extrastart<extraend):
4462 extrafieldslist.append(preheaderdata[extrastart]);
4463 extrastart = extrastart + 1;
4464 prefcs = preheaderdata[-2].lower();
4465 prenewfcs = preheaderdata[-1].lower();
4466 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4467 if(prefcs!=prenewfcs and not skipchecksum):
4468 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4469 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4470 return False;
4471 valid_archive = False;
4472 invalid_archive = True;
4473 prefhend = catfp.tell() - 1;
4474 prefcontentstart = catfp.tell();
4475 prefcontents = "";
4476 pyhascontents = False;
4477 if(prefsize>0):
4478 if(prefcompression):
4479 prefcontents = catfp.read(prefsize);
4480 else:
4481 prefcontents = catfp.read(prefcsize);
4482 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4483 pyhascontents = True;
4484 if(prefccs!=prenewfccs and not skipchecksum):
4485 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4486 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4487 return False;
4488 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4489 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4490 if(abs(fseeknextasnum)==0):
4491 pass;
4492 catfp.seek(fseeknextasnum, 1);
4493 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4494 fseeknextasnum = int(prefseeknextfile);
4495 if(abs(fseeknextasnum)==0):
4496 pass;
4497 catfp.seek(fseeknextasnum, 1);
4498 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4499 fseeknextasnum = int(prefseeknextfile);
4500 if(abs(fseeknextasnum)==0):
4501 pass;
4502 catfp.seek(fseeknextasnum, 0);
4503 else:
4504 return False;
4505 il = il + 1;
4506 filefound = False;
4507 prefname = preheaderdata[2];
4508 if(re.findall(r"^[.|/]", preheaderdata[2])):
4509 prefname = preheaderdata[2];
4510 else:
4511 prefname = "./"+preheaderdata[2];
4512 if(prefname==seekfile):
4513 filefound = True;
4514 break;
4515 catfp.seek(seekstart, 0);
4516 fileidnum = il;
4517 catfheadsize = int(preheaderdata[0], 16);
4518 catfnumfields = int(preheaderdata[1], 16);
4519 catftype = int(preheaderdata[2], 16);
4520 if(re.findall(r"^[.|/]", preheaderdata[3])):
4521 catfname = preheaderdata[3];
4522 else:
4523 catfname = "./"+preheaderdata[3];
4524 catflinkname = preheaderdata[4];
4525 catfsize = int(preheaderdata[5], 16);
4526 catfbasedir = os.path.dirname(catfname);
4527 if(filefound):
4528 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4529 else:
4530 return False;
4531 if(returnfp):
4532 catlist.update({'catfp': catfp});
4533 else:
4534 catfp.close();
4535 return catlist;
4536
4537 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4538
4539 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4540 formatspecs = FormatSpecsListToDict(formatspecs);
4541 if(verbose):
4542 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4543 if(hasattr(infile, "read") or hasattr(infile, "write")):
4544 catfp = infile;
4545 catfp.seek(0, 0);
4546 catfp = UncompressArchiveFile(catfp, formatspecs);
4547 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4548 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4549 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4550 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4551 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4552 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4553 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4554 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4555 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4556 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4557 return False;
4558 if(not catfp):
4559 return False;
4560 catfp.seek(0, 0);
4561 elif(infile=="-"):
4562 catfp = BytesIO();
4563 if(hasattr(sys.stdin, "buffer")):
4564 shutil.copyfileobj(sys.stdin.buffer, catfp);
4565 else:
4566 shutil.copyfileobj(sys.stdin, catfp);
4567 catfp.seek(0, 0);
4568 catfp = UncompressArchiveFile(catfp, formatspecs);
4569 if(not catfp):
4570 return False;
4571 catfp.seek(0, 0);
4572 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4573 catfp = download_file_from_internet_file(infile);
4574 catfp = UncompressArchiveFile(catfp, formatspecs);
4575 catfp.seek(0, 0);
4576 if(not catfp):
4577 return False;
4578 catfp.seek(0, 0);
4579 else:
4580 infile = RemoveWindowsPath(infile);
4581 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4582 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4583 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4584 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4585 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4586 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4587 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4588 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4589 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4590 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4591 return False;
4592 compresscheck = CheckCompressionType(infile, formatspecs, True);
4593 if(not compresscheck):
4594 fextname = os.path.splitext(infile)[1];
4595 if(fextname==".gz"):
4596 compresscheck = "gzip";
4597 elif(fextname==".bz2"):
4598 compresscheck = "bzip2";
4599 elif(fextname==".zst"):
4600 compresscheck = "zstd";
4601 elif(fextname==".lz4" or fextname==".clz4"):
4602 compresscheck = "lz4";
4603 elif(fextname==".lzo" or fextname==".lzop"):
4604 compresscheck = "lzo";
4605 elif(fextname==".lzma" or fextname==".xz"):
4606 compresscheck = "lzma";
4607 else:
4608 return False;
4609 if(not compresscheck):
4610 return False;
4611 catfp = UncompressFile(infile, formatspecs, "rb");
4612 '''
4613 try:
4614 catfp.seek(0, 2);
4615 except OSError:
4616 SeekToEndOfFile(catfp);
4617 except ValueError:
4618 SeekToEndOfFile(catfp);
4619 CatSize = catfp.tell();
4620 CatSizeEnd = CatSize;
4621 '''
4622 try:
4623 catfp.seek(0, 0);
4624 except OSError:
4625 return False;
4626 except ValueError:
4627 return False;
4628 curloc = catfp.tell();
4629 if(curloc>0):
4630 catfp.seek(0, 0);
4631 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4632 if(curloc>0):
4633 catfp.seek(curloc, 0);
4634 catstring = catheader[0];
4635 catversion = re.findall(r"([\d]+)", catstring);
4636 fprenumfiles = catheader[1];
4637 fnumfiles = int(fprenumfiles, 16);
4638 fprechecksumtype = catheader[2];
4639 fprechecksum = catheader[3];
4640 il = 0;
4641 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4642 fnumfileshex = format(int(fnumfiles), 'x').lower();
4643 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4644 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4645 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4646 valid_archive = True;
4647 invalid_archive = False;
4648 if(verbose):
4649 VerbosePrintOut(infile);
4650 VerbosePrintOut("Number of Records " + str(fnumfiles));
4651 if(fprechecksum==catfileheadercshex):
4652 if(verbose):
4653 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4654 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4655 else:
4656 if(verbose):
4657 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4658 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4659 valid_archive = False;
4660 invalid_archive = True;
4661 if(verbose):
4662 VerbosePrintOut("");
4663 while(il<fnumfiles):
4664 catfhstart = catfp.tell();
4665 if(formatspecs['new_style']):
4666 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4667 else:
4668 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4669 if(len(catheaderdata)==0):
4670 break;
4671 catfheadsize = int(catheaderdata[0], 16);
4672 catfnumfields = int(catheaderdata[1], 16);
4673 catftype = int(catheaderdata[2], 16);
4674 if(re.findall(r"^[.|/]", catheaderdata[3])):
4675 catfname = catheaderdata[3];
4676 else:
4677 catfname = "./"+catheaderdata[3];
4678 catfbasedir = os.path.dirname(catfname);
4679 catflinkname = catheaderdata[4];
4680 catfsize = int(catheaderdata[5], 16);
4681 catfatime = int(catheaderdata[6], 16);
4682 catfmtime = int(catheaderdata[7], 16);
4683 catfctime = int(catheaderdata[8], 16);
4684 catfbtime = int(catheaderdata[9], 16);
4685 catfmode = int(catheaderdata[10], 16);
4686 catfchmode = stat.S_IMODE(catfmode);
4687 catftypemod = stat.S_IFMT(catfmode);
4688 catfwinattributes = int(catheaderdata[11], 16);
4689 catfcompression = catheaderdata[12];
4690 catfcsize = int(catheaderdata[13], 16);
4691 catfuid = int(catheaderdata[14], 16);
4692 catfuname = catheaderdata[15];
4693 catfgid = int(catheaderdata[16], 16);
4694 catfgname = catheaderdata[17];
4695 fid = int(catheaderdata[18], 16);
4696 finode = int(catheaderdata[19], 16);
4697 flinkcount = int(catheaderdata[20], 16);
4698 catfdev_minor = int(catheaderdata[21], 16);
4699 catfdev_major = int(catheaderdata[22], 16);
4700 catfrdev_minor = int(catheaderdata[23], 16);
4701 catfrdev_major = int(catheaderdata[24], 16);
4702 catfseeknextfile = catheaderdata[25];
4703 catfextrasize = int(catheaderdata[26], 16);
4704 catfextrafields = int(catheaderdata[27], 16);
4705 extrafieldslist = [];
4706 extrastart = 28;
4707 extraend = extrastart + catfextrafields;
4708 extrafieldslist = [];
4709 if(extrastart<extraend):
4710 extrafieldslist.append(catheaderdata[extrastart]);
4711 extrastart = extrastart + 1;
4712 catfcs = catheaderdata[-2].lower();
4713 catfccs = catheaderdata[-1].lower();
4714 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4715 if(verbose):
4716 VerbosePrintOut(catfname);
4717 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4718 if(catfcs==catnewfcs):
4719 if(verbose):
4720 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4721 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4722 else:
4723 if(verbose):
4724 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4725 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4726 valid_archive = False;
4727 invalid_archive = True;
4728 catfhend = catfp.tell() - 1;
4729 catfcontentstart = catfp.tell();
4730 catfcontents = "";
4731 pyhascontents = False;
4732 if(catfsize>0):
4733 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4734 catfcontents = catfp.read(catfsize);
4735 else:
4736 catfcontents = catfp.read(catfcsize);
4737 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4738 pyhascontents = True;
4739 if(catfccs==catnewfccs):
4740 if(verbose):
4741 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4742 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4743 else:
4744 if(verbose):
4745 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4746 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4747 valid_archive = False;
4748 invalid_archive = True;
4749 if(verbose):
4750 VerbosePrintOut("");
4751 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
4752 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
4753 if(abs(fseeknextasnum)==0):
4754 pass;
4755 catfp.seek(fseeknextasnum, 1);
4756 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
4757 fseeknextasnum = int(catfseeknextfile);
4758 if(abs(fseeknextasnum)==0):
4759 pass;
4760 catfp.seek(fseeknextasnum, 1);
4761 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
4762 fseeknextasnum = int(catfseeknextfile);
4763 if(abs(fseeknextasnum)==0):
4764 pass;
4765 catfp.seek(fseeknextasnum, 0);
4766 else:
4767 return False;
4768 il = il + 1;
4769 if(valid_archive):
4770 if(returnfp):
4771 return catfp;
4772 else:
4773 catfp.close();
4774 return True;
4775 else:
4776 catfp.close();
4777 return False;
4778
4779 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4780
4781 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4782 formatspecs = FormatSpecsListToDict(formatspecs);
4783 if(hasattr(infile, "read") or hasattr(infile, "write")):
4784 catfp = infile;
4785 catfp.seek(0, 0);
4786 catfp = UncompressArchiveFile(catfp, formatspecs);
4787 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4788 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4789 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4790 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4791 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4792 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4793 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4794 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4795 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4796 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4797 return False;
4798 if(not catfp):
4799 return False;
4800 catfp.seek(0, 0);
4801 elif(infile=="-"):
4802 catfp = BytesIO();
4803 if(hasattr(sys.stdin, "buffer")):
4804 shutil.copyfileobj(sys.stdin.buffer, catfp);
4805 else:
4806 shutil.copyfileobj(sys.stdin, catfp);
4807 catfp.seek(0, 0);
4808 catfp = UncompressArchiveFile(catfp, formatspecs);
4809 if(not catfp):
4810 return False;
4811 catfp.seek(0, 0);
4812 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4813 catfp = download_file_from_internet_file(infile);
4814 catfp = UncompressArchiveFile(catfp, formatspecs);
4815 catfp.seek(0, 0);
4816 if(not catfp):
4817 return False;
4818 catfp.seek(0, 0);
4819 else:
4820 infile = RemoveWindowsPath(infile);
4821 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4822 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4823 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4824 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4825 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4826 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4827 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4828 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4829 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4830 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4831 return False;
4832 compresscheck = CheckCompressionType(infile, formatspecs, True);
4833 if(not compresscheck):
4834 fextname = os.path.splitext(infile)[1];
4835 if(fextname==".gz"):
4836 compresscheck = "gzip";
4837 elif(fextname==".bz2"):
4838 compresscheck = "bzip2";
4839 elif(fextname==".zst"):
4840 compresscheck = "zstd";
4841 elif(fextname==".lz4" or fextname==".clz4"):
4842 compresscheck = "lz4";
4843 elif(fextname==".lzo" or fextname==".lzop"):
4844 compresscheck = "lzo";
4845 elif(fextname==".lzma" or fextname==".xz"):
4846 compresscheck = "lzma";
4847 else:
4848 return False;
4849 if(not compresscheck):
4850 return False;
4851 catfp = UncompressFile(infile, formatspecs, "rb");
4852 '''
4853 try:
4854 catfp.seek(0, 2);
4855 except OSError:
4856 SeekToEndOfFile(catfp);
4857 except ValueError:
4858 SeekToEndOfFile(catfp);
4859 CatSize = catfp.tell();
4860 CatSizeEnd = CatSize;
4861 '''
4862 try:
4863 catfp.seek(0, 0);
4864 except OSError:
4865 return False;
4866 except ValueError:
4867 return False;
4868 curloc = catfp.tell();
4869 if(curloc>0):
4870 catfp.seek(0, 0);
4871 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4872 if(curloc>0):
4873 catfp.seek(curloc, 0);
4874 catstring = catheader[0];
4875 catversion = re.findall(r"([\d]+)", catstring);
4876 fprenumfiles = catheader[1];
4877 fnumfiles = int(fprenumfiles, 16);
4878 fprechecksumtype = catheader[2];
4879 fprechecksum = catheader[3];
4880 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4881 fnumfileshex = format(int(fnumfiles), 'x').lower();
4882 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4883 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4884 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4885 fheadtell = len(fileheader);
4886 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4887 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4888 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4889 return False;
4890 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4891 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
4892 if(seekstart<0 and seekstart>fnumfiles):
4893 seekstart = 0;
4894 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4895 seekend = fnumfiles;
4896 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4897 seekend = fnumfiles - abs(seekend);
4898 if(seekstart>0):
4899 il = 0;
4900 while(il < seekstart):
4901 prefhstart = catfp.tell();
4902 if(formatspecs['new_style']):
4903 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4904 else:
4905 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4906 if(len(preheaderdata)==0):
4907 break;
4908 prefheadsize = int(preheaderdata[0], 16);
4909 prefnumfields = int(preheaderdata[1], 16);
4910 if(re.findall(r"^[.|/]", preheaderdata[3])):
4911 prefname = preheaderdata[3];
4912 else:
4913 prefname = "./"+preheaderdata[3];
4914 prefsize = int(preheaderdata[5], 16);
4915 prefcompression = preheaderdata[12];
4916 prefcsize = int(preheaderdata[13], 16);
4917 prefseeknextfile = preheaderdata[25];
4918 prefextrasize = int(preheaderdata[26], 16);
4919 prefextrafields = int(preheaderdata[27], 16);
4920 extrafieldslist = [];
4921 extrastart = 28;
4922 extraend = extrastart + prefextrafields;
4923 extrafieldslist = [];
4924 if(extrastart<extraend):
4925 extrafieldslist.append(preheaderdata[extrastart]);
4926 extrastart = extrastart + 1;
4927 prefcs = preheaderdata[-2].lower();
4928 prenewfcs = preheaderdata[-1].lower();
4929 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4930 if(prefcs!=prenewfcs and not skipchecksum):
4931 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4932 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4933 return False;
4934 valid_archive = False;
4935 invalid_archive = True;
4936 prefhend = catfp.tell() - 1;
4937 prefcontentstart = catfp.tell();
4938 prefcontents = "";
4939 pyhascontents = False;
4940 if(prefsize>0):
4941 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4942 prefcontents = catfp.read(prefsize);
4943 else:
4944 prefcontents = catfp.read(prefcsize);
4945 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4946 pyhascontents = True;
4947 if(prefccs!=prenewfccs and not skipchecksum):
4948 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4949 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4950 return False;
4951 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4952 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4953 if(abs(fseeknextasnum)==0):
4954 pass;
4955 catfp.seek(fseeknextasnum, 1);
4956 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4957 fseeknextasnum = int(prefseeknextfile);
4958 if(abs(fseeknextasnum)==0):
4959 pass;
4960 catfp.seek(fseeknextasnum, 1);
4961 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4962 fseeknextasnum = int(prefseeknextfile);
4963 if(abs(fseeknextasnum)==0):
4964 pass;
4965 catfp.seek(fseeknextasnum, 0);
4966 else:
4967 return False;
4968 il = il + 1;
4969 fileidnum = seekstart;
4970 realidnum = 0;
4971 while(fileidnum<seekend):
4972 catfhstart = catfp.tell();
4973 if(formatspecs['new_style']):
4974 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4975 else:
4976 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4977 if(len(catheaderdata)==0):
4978 break;
4979 catfheadsize = int(catheaderdata[0], 16);
4980 catfnumfields = int(catheaderdata[1], 16);
4981 catftype = int(catheaderdata[2], 16);
4982 if(re.findall(r"^[.|/]", catheaderdata[3])):
4983 catfname = catheaderdata[3];
4984 else:
4985 catfname = "./"+catheaderdata[3];
4986 catfbasedir = os.path.dirname(catfname);
4987 catflinkname = catheaderdata[4];
4988 catfsize = int(catheaderdata[5], 16);
4989 catfatime = int(catheaderdata[6], 16);
4990 catfmtime = int(catheaderdata[7], 16);
4991 catfctime = int(catheaderdata[8], 16);
4992 catfbtime = int(catheaderdata[9], 16);
4993 catfmode = int(catheaderdata[10], 16);
4994 catfchmode = stat.S_IMODE(catfmode);
4995 catftypemod = stat.S_IFMT(catfmode);
4996 catfwinattributes = int(catheaderdata[11], 16);
4997 catfcompression = catheaderdata[12];
4998 catfcsize = int(catheaderdata[13], 16);
4999 catfuid = int(catheaderdata[14], 16);
5000 catfuname = catheaderdata[15];
5001 catfgid = int(catheaderdata[16], 16);
5002 catfgname = catheaderdata[17];
5003 catfid = int(catheaderdata[18], 16);
5004 catfinode = int(catheaderdata[19], 16);
5005 catflinkcount = int(catheaderdata[20], 16);
5006 catfdev_minor = int(catheaderdata[21], 16);
5007 catfdev_major = int(catheaderdata[22], 16);
5008 catfrdev_minor = int(catheaderdata[23], 16);
5009 catfrdev_major = int(catheaderdata[24], 16);
5010 catfseeknextfile = catheaderdata[25];
5011 catfextrasize = int(catheaderdata[26], 16);
5012 catfextrafields = int(catheaderdata[27], 16);
5013 extrafieldslist = [];
5014 extrastart = 28;
5015 extraend = extrastart + catfextrafields;
5016 extrafieldslist = [];
5017 if(extrastart<extraend):
5018 extrafieldslist.append(catheaderdata[extrastart]);
5019 extrastart = extrastart + 1;
5020 catfcs = catheaderdata[-2].lower();
5021 catfccs = catheaderdata[-1].lower();
5022 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5023 if(catfcs!=catnewfcs and not skipchecksum):
5024 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
5025 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5026 return False;
5027 catfhend = catfp.tell() - 1;
5028 catfcontentstart = catfp.tell();
5029 catfcontents = BytesIO();
5030 pyhascontents = False;
5031 if(catfsize>0 and not listonly):
5032 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5033 catfcontents.write(catfp.read(catfsize));
5034 else:
5035 catfcontents.write(catfp.read(catfcsize));
5036 catfcontents.seek(0, 0);
5037 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5038 pyhascontents = True;
5039 if(catfccs!=catnewfccs and skipchecksum):
5040 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5041 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5042 return False;
5043 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5044 pass;
5045 else:
5046 catfcontents.seek(0, 0);
5047 if(uncompress):
5048 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5049 catfcontents.seek(0, 0);
5050 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5051 if(catfsize>0 and listonly):
5052 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5053 catfp.seek(catfsize, 1);
5054 else:
5055 catfp.seek(catfcsize, 1);
5056 pyhascontents = False;
5057 catfcontentend = catfp.tell();
5058 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5059 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5060 if(abs(fseeknextasnum)==0):
5061 pass;
5062 catfp.seek(fseeknextasnum, 1);
5063 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5064 fseeknextasnum = int(catfseeknextfile);
5065 if(abs(fseeknextasnum)==0):
5066 pass;
5067 catfp.seek(fseeknextasnum, 1);
5068 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5069 fseeknextasnum = int(catfseeknextfile);
5070 if(abs(fseeknextasnum)==0):
5071 pass;
5072 catfp.seek(fseeknextasnum, 0);
5073 else:
5074 return False;
5075 catfcontents.seek(0, 0);
5076 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
5077 fileidnum = fileidnum + 1;
5078 realidnum = realidnum + 1;
5079 if(returnfp):
5080 catlist.update({'catfp': catfp});
5081 else:
5082 catfp.close();
5083 return catlist;
5084
5085 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5086
5087 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5088 formatspecs = FormatSpecsListToDict(formatspecs);
5089 catfp = BytesIO(catstr);
5090 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5091 return listcatfiles;
5092
5093 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5094
5095 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5096 formatspecs = FormatSpecsListToDict(formatspecs);
5097 catfp = BytesIO();
5098 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5099 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5100 return listcatfiles;
5101
5102 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5103 formatspecs = FormatSpecsListToDict(formatspecs);
5104 catfp = BytesIO();
5105 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5106 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5107 return listcatfiles;
5108
5109 if(not rarfile_support):
5110 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5111 return False;
5112
5113 if(rarfile_support):
5114 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5115 formatspecs = FormatSpecsListToDict(formatspecs);
5116 catfp = BytesIO();
5117 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5118 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5119 return listcatfiles;
5120
5121 if(not py7zr_support):
5122 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5123 return False;
5124
5125 if(py7zr_support):
5126 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5127 formatspecs = FormatSpecsListToDict(formatspecs);
5128 catfp = BytesIO();
5129 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5130 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5131 return listcatfiles;
5132
5133 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5134 formatspecs = FormatSpecsListToDict(formatspecs);
5135 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5136 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5137 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5138 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5139 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5140 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5141 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5142 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5143 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5144 elif(checkcompressfile=="catfile"):
5145 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5146 else:
5147 return False;
5148 return False;
5149
5150 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5151 formatspecs = FormatSpecsListToDict(formatspecs);
5152 catver = formatspecs['format_ver'];
5153 fileheaderver = str(int(catver.replace(".", "")));
5154 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5155 advancedlist = formatspecs['use_advanced_list'];
5156 altinode = formatspecs['use_alt_inode'];
5157 infilelist = [];
5158 if(infiles=="-"):
5159 for line in sys.stdin:
5160 infilelist.append(line.strip());
5161 infilelist = list(filter(None, infilelist));
5162 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5163 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5164 return False;
5165 with UncompressFile(infiles, formatspecs, "r") as finfile:
5166 for line in finfile:
5167 infilelist.append(line.strip());
5168 infilelist = list(filter(None, infilelist));
5169 else:
5170 if(isinstance(infiles, (list, tuple, ))):
5171 infilelist = list(filter(None, infiles));
5172 elif(isinstance(infiles, (str, ))):
5173 infilelist = list(filter(None, [infiles]));
5174 if(advancedlist):
5175 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5176 else:
5177 GetDirList = ListDir(infilelist, followlink, False);
5178 if(not GetDirList):
5179 return False;
5180 curinode = 0;
5181 curfid = 0;
5182 inodelist = [];
5183 inodetofile = {};
5184 filetoinode = {};
5185 inodetocatinode = {};
5186 fileidnum = 0;
5187 fnumfiles = int(len(GetDirList));
5188 catver = formatspecs['format_ver'];
5189 fileheaderver = str(int(catver.replace(".", "")));
5190 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5191 fnumfileshex = format(int(fnumfiles), 'x').lower();
5192 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5193 catversion = re.findall(r"([\d]+)", fileheader);
5194 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5195 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5196 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5197 fheadtell = len(fileheader);
5198 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5199 for curfname in GetDirList:
5200 catfhstart = fheadtell;
5201 if(re.findall(r"^[.|/]", curfname)):
5202 fname = curfname;
5203 else:
5204 fname = "./"+curfname;
5205 if(verbose):
5206 VerbosePrintOut(fname);
5207 if(not followlink or followlink is None):
5208 fstatinfo = os.lstat(fname);
5209 else:
5210 fstatinfo = os.stat(fname);
5211 fpremode = fstatinfo.st_mode;
5212 finode = fstatinfo.st_ino;
5213 flinkcount = fstatinfo.st_nlink;
5214 ftype = 0;
5215 if(stat.S_ISREG(fpremode)):
5216 ftype = 0;
5217 elif(stat.S_ISLNK(fpremode)):
5218 ftype = 2;
5219 elif(stat.S_ISCHR(fpremode)):
5220 ftype = 3;
5221 elif(stat.S_ISBLK(fpremode)):
5222 ftype = 4;
5223 elif(stat.S_ISDIR(fpremode)):
5224 ftype = 5;
5225 elif(stat.S_ISFIFO(fpremode)):
5226 ftype = 6;
5227 elif(stat.S_ISSOCK(fpremode)):
5228 ftype = 8;
5229 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5230 ftype = 9;
5231 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5232 ftype = 10;
5233 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5234 ftype = 11;
5235 else:
5236 ftype = 0;
5237 flinkname = "";
5238 fbasedir = os.path.dirname(fname);
5239 fcurfid = curfid;
5240 if(not followlink and finode!=0):
5241 if(ftype!=1):
5242 if(finode in inodelist):
5243 ftype = 1;
5244 flinkname = inodetofile[finode];
5245 if(altinode):
5246 fcurinode = finode;
5247 else:
5248 fcurinode = inodetocatinode[finode];
5249 if(finode not in inodelist):
5250 inodelist.append(finode);
5251 inodetofile.update({finode: fname});
5252 inodetocatinode.update({finode: curinode});
5253 if(altinode):
5254 fcurinode = finode;
5255 else:
5256 fcurinode = curinode;
5257 curinode = curinode + 1;
5258 else:
5259 fcurinode = curinode;
5260 curinode = curinode + 1;
5261 curfid = curfid + 1;
5262 if(ftype==2):
5263 flinkname = os.readlink(fname);
5264 fdev = fstatinfo.st_dev;
5265 getfdev = GetDevMajorMinor(fdev);
5266 fdev_minor = getfdev[0];
5267 fdev_major = getfdev[1];
5268 frdev = fstatinfo.st_dev;
5269 if(hasattr(fstatinfo, "st_rdev")):
5270 frdev = fstatinfo.st_rdev;
5271 else:
5272 frdev = fstatinfo.st_dev;
5273 getfrdev = GetDevMajorMinor(frdev);
5274 frdev_minor = getfrdev[0];
5275 frdev_major = getfrdev[1];
5276 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5277 fsize = "0";
5278 if(ftype==0 or ftype==7):
5279 fsize = fstatinfo.st_size;
5280 fatime = fstatinfo.st_atime;
5281 fmtime = fstatinfo.st_mtime;
5282 fctime = fstatinfo.st_ctime;
5283 if(hasattr(fstatinfo, "st_birthtime")):
5284 fbtime = fstatinfo.st_birthtime;
5285 else:
5286 fbtime = fstatinfo.st_ctime;
5287 fmode = fstatinfo.st_mode;
5288 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5289 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5290 fuid = fstatinfo.st_uid;
5291 fgid = fstatinfo.st_gid;
5292 funame = "";
5293 try:
5294 import pwd;
5295 try:
5296 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5297 funame = userinfo.pw_name;
5298 except KeyError:
5299 funame = "";
5300 except ImportError:
5301 funame = "";
5302 fgname = "";
5303 try:
5304 import grp;
5305 try:
5306 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5307 fgname = groupinfo.gr_name;
5308 except KeyError:
5309 fgname = "";
5310 except ImportError:
5311 fgname = "";
5312 fdev_minor = fdev_minor;
5313 fdev_major = fdev_major;
5314 frdev_minor = frdev_minor;
5315 frdev_major = frdev_major;
5316 flinkcount = flinkcount;
5317 if(hasattr(fstatinfo, "st_file_attributes")):
5318 fwinattributes = fstatinfo.st_file_attributes;
5319 else:
5320 fwinattributes = 0;
5321 fcompression = "";
5322 fcsize = 0;
5323 fcontents = BytesIO();
5324 if(ftype==0 or ftype==7):
5325 with open(fname, "rb") as fpc:
5326 shutil.copyfileobj(fpc, fcontents);
5327 if(followlink and (ftype==1 or ftype==2)):
5328 flstatinfo = os.stat(flinkname);
5329 with open(flinkname, "rb") as fpc:
5330 shutil.copyfileobj(fpc, fcontents);
5331 fcontents.seek(0, 0);
5332 ftypehex = format(ftype, 'x').lower();
5333 extrafields = len(extradata);
5334 extrafieldslist = extradata;
5335 catfextrafields = extrafields;
5336 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5337 if(len(extradata)>0):
5338 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5339 extrasizelen = len(extrasizestr);
5340 extrasizelenhex = format(extrasizelen, 'x').lower();
5341 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5342 catoutlen = len(catoutlist) + len(extradata) + 3;
5343 catoutlenhex = format(catoutlen, 'x').lower();
5344 catoutlist.insert(0, catoutlenhex);
5345 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5346 catheaderdata = catoutlist;
5347 if(len(extradata)>0):
5348 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5349 if(fsize==0):
5350 checksumlist = [checksumtype, "none"];
5351 else:
5352 checksumlist = [checksumtype, checksumtype];
5353 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5354 catfnumfields = catoutlen;
5355 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5356 fcontents.seek(0, 0);
5357 if(fsize==0):
5358 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5359 else:
5360 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5361 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5362 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5363 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5364 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5365 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5366 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5367 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5368 catfcontentstart = fheadtell;
5369 fheadtell += len(catfileoutstr) + 1;
5370 catfcontentend = fheadtell - 1;
5371 catfhend = catfcontentend;
5372 fcontents.seek(0, 0);
5373 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5374 pyhascontents = False;
5375 if(int(fsize)>0 and not listonly):
5376 pyhascontents = True;
5377 if(int(fsize)>0 and listonly):
5378 fcontents = BytesIO();
5379 pyhascontents = False;
5380 fcontents.seek(0, 0);
5381 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5382 fileidnum = fileidnum + 1;
5383 return catlist;
5384
5385 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5386 formatspecs = FormatSpecsListToDict(formatspecs);
5387 curinode = 0;
5388 curfid = 0;
5389 inodelist = [];
5390 inodetofile = {};
5391 filetoinode = {};
5392 inodetocatinode = {};
5393 fileidnum = 0;
5394 if(infile=="-"):
5395 infile = BytesIO();
5396 if(hasattr(sys.stdin, "buffer")):
5397 shutil.copyfileobj(sys.stdin.buffer, infile);
5398 else:
5399 shutil.copyfileobj(sys.stdin, infile);
5400 infile.seek(0, 0);
5401 if(not infile):
5402 return False;
5403 infile.seek(0, 0);
5404 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5405 infile = download_file_from_internet_file(infile);
5406 infile.seek(0, 0);
5407 if(not infile):
5408 return False;
5409 infile.seek(0, 0);
5410 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5411 return False;
5412 elif(os.path.exists(infile) and os.path.isfile(infile)):
5413 try:
5414 if(not tarfile.TarFileCheck(infile)):
5415 return False;
5416 except AttributeError:
5417 if(not TarFileCheck(infile)):
5418 return False;
5419 try:
5420 if(hasattr(infile, "read") or hasattr(infile, "write")):
5421 tarfp = tarfile.open(fileobj=infile, mode="r");
5422 else:
5423 tarfp = tarfile.open(infile, "r");
5424 except FileNotFoundError:
5425 return False;
5426 fnumfiles = int(len(tarfp.getmembers()));
5427 catver = formatspecs['format_ver'];
5428 fileheaderver = str(int(catver.replace(".", "")));
5429 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5430 fnumfileshex = format(int(fnumfiles), 'x').lower();
5431 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5432 catversion = re.findall(r"([\d]+)", fileheader);
5433 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5434 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5435 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5436 fheadtell = len(fileheader);
5437 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5438 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5439 catfhstart = fheadtell;
5440 if(re.findall(r"^[.|/]", member.name)):
5441 fname = member.name;
5442 else:
5443 fname = "./"+member.name;
5444 if(verbose):
5445 VerbosePrintOut(fname);
5446 fpremode = member.mode;
5447 ffullmode = member.mode;
5448 flinkcount = 0;
5449 ftype = 0;
5450 if(member.isreg()):
5451 ffullmode = member.mode + stat.S_IFREG;
5452 ftype = 0;
5453 elif(member.isdev()):
5454 ffullmode = member.mode;
5455 ftype = 7;
5456 elif(member.islnk()):
5457 ffullmode = member.mode + stat.S_IFREG;
5458 ftype = 1;
5459 elif(member.issym()):
5460 ffullmode = member.mode + stat.S_IFLNK;
5461 ftype = 2;
5462 elif(member.ischr()):
5463 ffullmode = member.mode + stat.S_IFCHR;
5464 ftype = 3;
5465 elif(member.isblk()):
5466 ffullmode = member.mode + stat.S_IFBLK;
5467 ftype = 4;
5468 elif(member.isdir()):
5469 ffullmode = member.mode + stat.S_IFDIR;
5470 ftype = 5;
5471 elif(member.isfifo()):
5472 ffullmode = member.mode + stat.S_IFIFO;
5473 ftype = 6;
5474 elif(member.issparse()):
5475 ffullmode = member.mode;
5476 ftype = 12;
5477 else:
5478 ffullmode = member.mode;
5479 ftype = 0;
5480 flinkname = "";
5481 fbasedir = os.path.dirname(fname);
5482 fcurfid = curfid;
5483 fcurinode = curfid;
5484 finode = fcurinode;
5485 curfid = curfid + 1;
5486 if(ftype==2):
5487 flinkname = member.linkname;
5488 fdev_minor = member.devminor;
5489 fdev_major = member.devmajor;
5490 frdev_minor = member.devminor;
5491 frdev_major = member.devmajor;
5492 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5493 fsize = "0";
5494 elif(ftype==0 or ftype==7):
5495 fsize = member.size;
5496 else:
5497 fsize = member.size;
5498 fatime = member.mtime;
5499 fmtime = member.mtime;
5500 fctime = member.mtime;
5501 fbtime = member.mtime;
5502 fmode = ffullmode;
5503 fchmode = stat.S_IMODE(ffullmode);
5504 ftypemod = stat.S_IFMT(ffullmode);
5505 fuid = member.uid;
5506 fgid = member.gid;
5507 funame = member.uname;
5508 fgname = member.gname;
5509 flinkcount = flinkcount;
5510 fwinattributes = int(0);
5511 fcompression = "";
5512 fcsize = 0;
5513 fcontents = BytesIO();
5514 if(ftype==0 or ftype==7):
5515 with tarfp.extractfile(member) as fpc:
5516 shutil.copyfileobj(fpc, fcontents);
5517 fcontents.seek(0, 0);
5518 ftypehex = format(ftype, 'x').lower();
5519 extrafields = len(extradata);
5520 extrafieldslist = extradata;
5521 catfextrafields = extrafields;
5522 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5523 if(len(extradata)>0):
5524 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5525 extrasizelen = len(extrasizestr);
5526 extrasizelenhex = format(extrasizelen, 'x').lower();
5527 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5528 catoutlen = len(catoutlist) + len(extradata) + 3;
5529 catoutlenhex = format(catoutlen, 'x').lower();
5530 catoutlist.insert(0, catoutlenhex);
5531 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5532 catheaderdata = catoutlist;
5533 if(len(extradata)>0):
5534 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5535 if(fsize==0):
5536 checksumlist = [checksumtype, "none"];
5537 else:
5538 checksumlist = [checksumtype, checksumtype];
5539 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5540 catfnumfields = catoutlen;
5541 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5542 fcontents.seek(0, 0);
5543 if(fsize==0):
5544 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5545 else:
5546 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5547 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5548 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5549 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5550 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5551 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5552 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5553 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5554 catfcontentstart = fheadtell;
5555 fheadtell += len(catfileoutstr) + 1;
5556 catfcontentend = fheadtell - 1;
5557 catfhend = catfcontentend;
5558 fcontents.seek(0, 0);
5559 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5560 pyhascontents = False;
5561 if(int(fsize)>0 and not listonly):
5562 pyhascontents = True;
5563 if(int(fsize)>0 and listonly):
5564 fcontents = BytesIO();
5565 pyhascontents = False;
5566 fcontents.seek(0, 0);
5567 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5568 fileidnum = fileidnum + 1;
5569 return catlist;
5570
5571 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5572 formatspecs = FormatSpecsListToDict(formatspecs);
5573 curinode = 0;
5574 curfid = 0;
5575 inodelist = [];
5576 inodetofile = {};
5577 filetoinode = {};
5578 inodetocatinode = {};
5579 fileidnum = 0;
5580 if(infile=="-"):
5581 infile = BytesIO();
5582 if(hasattr(sys.stdin, "buffer")):
5583 shutil.copyfileobj(sys.stdin.buffer, infile);
5584 else:
5585 shutil.copyfileobj(sys.stdin, infile);
5586 infile.seek(0, 0);
5587 if(not infile):
5588 return False;
5589 infile.seek(0, 0);
5590 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5591 infile = download_file_from_internet_file(infile);
5592 infile.seek(0, 0);
5593 if(not infile):
5594 return False;
5595 infile.seek(0, 0);
5596 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5597 return False;
5598 if(not zipfile.is_zipfile(infile)):
5599 return False;
5600 try:
5601 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5602 except FileNotFoundError:
5603 return False;
5604 ziptest = zipfp.testzip();
5605 if(ziptest):
5606 VerbosePrintOut("Bad file found!");
5607 fnumfiles = int(len(zipfp.infolist()));
5608 catver = formatspecs['format_ver'];
5609 fileheaderver = str(int(catver.replace(".", "")));
5610 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5611 catversion = re.findall(r"([\d]+)", fileheader);
5612 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5613 fnumfileshex = format(int(fnumfiles), 'x').lower();
5614 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5615 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5616 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5617 fheadtell = len(fileheader);
5618 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5619 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5620 catfhstart = fheadtell;
5621 if(re.findall(r"^[.|/]", member.filename)):
5622 fname = member.filename;
5623 else:
5624 fname = "./"+member.filename;
5625 zipinfo = zipfp.getinfo(member.filename);
5626 if(verbose):
5627 VerbosePrintOut(fname);
5628 if(not member.is_dir()):
5629 fpremode = stat.S_IFREG + 438;
5630 elif(member.is_dir()):
5631 fpremode = stat.S_IFDIR + 511;
5632 flinkcount = 0;
5633 ftype = 0;
5634 if(not member.is_dir()):
5635 ftype = 0;
5636 elif(member.is_dir()):
5637 ftype = 5;
5638 flinkname = "";
5639 fbasedir = os.path.dirname(fname);
5640 fcurfid = curfid;
5641 fcurinode = curfid;
5642 finode = fcurinode;
5643 curfid = curfid + 1;
5644 fdev_minor = 0;
5645 fdev_major = 0;
5646 frdev_minor = 0;
5647 frdev_major = 0;
5648 if(ftype==5):
5649 fsize = "0";
5650 elif(ftype==0):
5651 fsize = member.file_size;
5652 else:
5653 fsize = member.file_size;
5654 fatime = time.mktime(member.date_time + (0, 0, -1));
5655 fmtime = time.mktime(member.date_time + (0, 0, -1));
5656 fctime = time.mktime(member.date_time + (0, 0, -1));
5657 fbtime = time.mktime(member.date_time + (0, 0, -1));
5658 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5659 fwinattributes = int(zipinfo.external_attr);
5660 if(not member.is_dir()):
5661 fmode = int(stat.S_IFREG + 438);
5662 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5663 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5664 elif(member.is_dir()):
5665 fmode = int(stat.S_IFDIR + 511);
5666 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5667 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5668 elif(zipinfo.create_system==3):
5669 fwinattributes = int(0);
5670 fmode = int(zipinfo.external_attr);
5671 else:
5672 fwinattributes = int(0);
5673 if(not member.is_dir()):
5674 fmode = int(stat.S_IFREG + 438);
5675 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5676 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5677 elif(member.is_dir()):
5678 fmode = int(stat.S_IFDIR + 511);
5679 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5680 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5681 fcompression = "";
5682 fcsize = 0;
5683 try:
5684 fuid = os.getuid();
5685 except AttributeError:
5686 fuid = 0;
5687 except KeyError:
5688 fuid = 0;
5689 try:
5690 fgid = os.getgid();
5691 except AttributeError:
5692 fgid = 0;
5693 except KeyError:
5694 fgid = 0;
5695 try:
5696 import pwd;
5697 try:
5698 userinfo = pwd.getpwuid(os.getuid());
5699 funame = userinfo.pw_name;
5700 except KeyError:
5701 funame = "";
5702 except AttributeError:
5703 funame = "";
5704 except ImportError:
5705 funame = "";
5706 fgname = "";
5707 try:
5708 import grp;
5709 try:
5710 groupinfo = grp.getgrgid(os.getgid());
5711 fgname = groupinfo.gr_name;
5712 except KeyError:
5713 fgname = "";
5714 except AttributeError:
5715 fgname = "";
5716 except ImportError:
5717 fgname = "";
5718 fcontents = BytesIO();
5719 if(ftype==0):
5720 fcontents.write(zipfp.read(member.filename));
5721 fcontents.seek(0, 0);
5722 ftypehex = format(ftype, 'x').lower();
5723 extrafields = len(extradata);
5724 extrafieldslist = extradata;
5725 catfextrafields = extrafields;
5726 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5727 if(len(extradata)>0):
5728 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5729 extrasizelen = len(extrasizestr);
5730 extrasizelenhex = format(extrasizelen, 'x').lower();
5731 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5732 catoutlen = len(catoutlist) + len(extradata) + 3;
5733 catoutlenhex = format(catoutlen, 'x').lower();
5734 catoutlist.insert(0, catoutlenhex);
5735 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5736 catheaderdata = catoutlist;
5737 if(len(extradata)>0):
5738 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5739 if(fsize==0):
5740 checksumlist = [checksumtype, "none"];
5741 else:
5742 checksumlist = [checksumtype, checksumtype];
5743 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5744 catfnumfields = catoutlen;
5745 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5746 fcontents.seek(0, 0);
5747 if(fsize==0):
5748 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5749 else:
5750 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5751 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5752 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5753 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5754 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5755 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5756 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5757 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5758 catfcontentstart = fheadtell;
5759 fheadtell += len(catfileoutstr) + 1;
5760 catfcontentend = fheadtell - 1;
5761 catfhend = catfcontentend;
5762 fcontents.seek(0, 0);
5763 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5764 pyhascontents = False;
5765 if(int(fsize)>0 and not listonly):
5766 pyhascontents = True;
5767 if(int(fsize)>0 and listonly):
5768 fcontents = BytesIO();
5769 pyhascontents = False;
5770 fcontents.seek(0, 0);
5771 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5772 fileidnum = fileidnum + 1;
5773 return catlist;
5774
5775 if(not rarfile_support):
5776 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5777 return False;
5778
5779 if(rarfile_support):
5780 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5781 formatspecs = FormatSpecsListToDict(formatspecs);
5782 curinode = 0;
5783 curfid = 0;
5784 inodelist = [];
5785 inodetofile = {};
5786 filetoinode = {};
5787 inodetocatinode = {};
5788 fileidnum = 0;
5789 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5790 return False;
5791 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5792 return False;
5793 rarfp = rarfile.RarFile(infile, "r");
5794 rartest = rarfp.testrar();
5795 if(rartest):
5796 VerbosePrintOut("Bad file found!");
5797 fnumfiles = int(len(rarfp.infolist()));
5798 catver = formatspecs['format_ver'];
5799 fileheaderver = str(int(catver.replace(".", "")));
5800 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5801 catversion = re.findall(r"([\d]+)", fileheader);
5802 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5803 fnumfileshex = format(int(fnumfiles), 'x').lower();
5804 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5805 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5806 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5807 fheadtell = len(fileheader);
5808 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5809 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5810 catfhstart = fheadtell;
5811 is_unix = False;
5812 is_windows = False;
5813 if(member.host_os==rarfile.RAR_OS_UNIX):
5814 is_windows = False;
5815 try:
5816 member.external_attr
5817 is_unix = True;
5818 except AttributeError:
5819 is_unix = False;
5820 elif(member.host_os==rarfile.RAR_OS_WIN32):
5821 is_unix = False;
5822 try:
5823 member.external_attr
5824 is_windows = True;
5825 except AttributeError:
5826 is_windows = False;
5827 else:
5828 is_unix = False;
5829 is_windows = False;
5830 if(re.findall(r"^[.|/]", member.filename)):
5831 fname = member.filename;
5832 else:
5833 fname = "./"+member.filename;
5834 rarinfo = rarfp.getinfo(member.filename);
5835 if(verbose):
5836 VerbosePrintOut(fname);
5837 if(is_unix and member.external_attr !=0):
5838 fpremode = int(member.external_attr);
5839 elif(member.is_file()):
5840 fpremode = stat.S_IFREG + 438;
5841 elif(member.is_symlink()):
5842 fpremode = stat.S_IFLNK + 438;
5843 elif(member.is_dir()):
5844 fpremode = stat.S_IFDIR + 511;
5845 if(is_windows and member.external_attr !=0):
5846 fwinattributes = int(member.external_attr);
5847 else:
5848 fwinattributes = int(0);
5849 fcompression = "";
5850 fcsize = 0;
5851 flinkcount = 0;
5852 ftype = 0;
5853 if(member.is_file()):
5854 ftype = 0;
5855 elif(member.is_symlink()):
5856 ftype = 2;
5857 elif(member.is_dir()):
5858 ftype = 5;
5859 flinkname = "";
5860 if(ftype==2):
5861 flinkname = rarfp.read(member.filename).decode("UTF-8");
5862 fbasedir = os.path.dirname(fname);
5863 fcurfid = curfid;
5864 fcurinode = curfid;
5865 finode = fcurinode;
5866 curfid = curfid + 1;
5867 fdev_minor = 0;
5868 fdev_major = 0;
5869 frdev_minor = 0;
5870 frdev_major = 0;
5871 if(ftype==5):
5872 fsize = "0";
5873 if(ftype==0):
5874 fsize = member.file_size;
5875 try:
5876 if(member.atime):
5877 fatime = int(member.atime.timestamp());
5878 else:
5879 fatime = int(member.mtime.timestamp());
5880 except AttributeError:
5881 fatime = int(member.mtime.timestamp());
5882 fmtime = int(member.mtime.timestamp());
5883 try:
5884 if(member.ctime):
5885 fctime = int(member.ctime.timestamp());
5886 else:
5887 fctime = int(member.mtime.timestamp());
5888 except AttributeError:
5889 fctime = int(member.mtime.timestamp());
5890 fbtime = int(member.mtime.timestamp());
5891 if(is_unix and member.external_attr !=0):
5892 fmode = int(member.external_attr);
5893 fchmode = int(stat.S_IMODE(member.external_attr));
5894 ftypemod = int(stat.S_IFMT(member.external_attr));
5895 elif(member.is_file()):
5896 fmode = int(stat.S_IFREG + 438)
5897 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5898 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5899 elif(member.is_symlink()):
5900 fmode = int(stat.S_IFLNK + 438)
5901 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5902 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5903 elif(member.is_dir()):
5904 fmode = int(stat.S_IFDIR + 511)
5905 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5906 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5907 try:
5908 fuid = os.getuid();
5909 except AttributeError:
5910 fuid = 0;
5911 except KeyError:
5912 fuid = 0;
5913 try:
5914 fgid = os.getgid();
5915 except AttributeError:
5916 fgid = 0;
5917 except KeyError:
5918 fgid = 0;
5919 try:
5920 import pwd;
5921 try:
5922 userinfo = pwd.getpwuid(os.getuid());
5923 funame = userinfo.pw_name;
5924 except KeyError:
5925 funame = "";
5926 except AttributeError:
5927 funame = "";
5928 except ImportError:
5929 funame = "";
5930 fgname = "";
5931 try:
5932 import grp;
5933 try:
5934 groupinfo = grp.getgrgid(os.getgid());
5935 fgname = groupinfo.gr_name;
5936 except KeyError:
5937 fgname = "";
5938 except AttributeError:
5939 fgname = "";
5940 except ImportError:
5941 fgname = "";
5942 fcontents = BytesIO();
5943 if(ftype==0):
5944 fcontents.write(rarfp.read(member.filename));
5945 fcontents.seek(0, 0);
5946 ftypehex = format(ftype, 'x').lower();
5947 extrafields = len(extradata);
5948 extrafieldslist = extradata;
5949 catfextrafields = extrafields;
5950 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5951 if(len(extradata)>0):
5952 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5953 extrasizelen = len(extrasizestr);
5954 extrasizelenhex = format(extrasizelen, 'x').lower();
5955 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5956 catoutlen = len(catoutlist) + len(extradata) + 3;
5957 catoutlenhex = format(catoutlen, 'x').lower();
5958 catoutlist.insert(0, catoutlenhex);
5959 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5960 if(len(extradata)>0):
5961 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5962 if(fsize==0):
5963 checksumlist = [checksumtype, "none"];
5964 else:
5965 checksumlist = [checksumtype, checksumtype];
5966 ccatfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5967 catfnumfields = 24 + catfextrafields;
5968 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5969 fcontents.seek(0, 0);
5970 if(fsize==0):
5971 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5972 else:
5973 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5974 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5975 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5976 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5977 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5978 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5979 catheaderdata = catoutlist;
5980 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5981 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5982 catfcontentstart = fheadtell;
5983 fheadtell += len(catfileoutstr) + 1;
5984 catfcontentend = fheadtell - 1;
5985 catfhend = catfcontentend;
5986 fcontents.seek(0, 0);
5987 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5988 pyhascontents = False;
5989 if(int(fsize)>0 and not listonly):
5990 pyhascontents = True;
5991 if(int(fsize)>0 and listonly):
5992 fcontents = BytesIO();
5993 pyhascontents = False;
5994 fcontents.seek(0, 0);
5995 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5996 fileidnum = fileidnum + 1;
5997 return catlist;
5998
5999 if(not py7zr_support):
6000 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6001 return False;
6002
6003 if(py7zr_support):
6004 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6005 formatspecs = FormatSpecsListToDict(formatspecs);
6006 curinode = 0;
6007 curfid = 0;
6008 inodelist = [];
6009 inodetofile = {};
6010 filetoinode = {};
6011 inodetocatinode = {};
6012 fileidnum = 0;
6013 szpfp = py7zr.SevenZipFile(infile, mode="r");
6014 file_content = szpfp.readall();
6015 #sztest = szpfp.testzip();
6016 sztestalt = szpfp.test();
6017 if(sztestalt):
6018 VerbosePrintOut("Bad file found!");
6019 numfiles = int(len(szpfp.list()));
6020 catver = formatspecs['format_ver'];
6021 fileheaderver = str(int(catver.replace(".", "")));
6022 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6023 catversion = re.findall(r"([\d]+)", fileheader);
6024 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6025 fnumfileshex = format(int(fnumfiles), 'x').lower();
6026 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6027 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6028 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6029 fheadtell = len(fileheader);
6030 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6031 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6032 catfhstart = fheadtell;
6033 if(re.findall(r"^[.|/]", member.filename)):
6034 fname = member.filename;
6035 else:
6036 fname = "./"+member.filename;
6037 if(not member.is_directory):
6038 fpremode = int(stat.S_IFREG + 438);
6039 elif(member.is_directory):
6040 fpremode = int(stat.S_IFDIR + 511);
6041 fwinattributes = int(0);
6042 fcompression = "";
6043 fcsize = 0;
6044 flinkcount = 0;
6045 ftype = 0;
6046 if(member.is_directory):
6047 ftype = 5;
6048 else:
6049 ftype = 0;
6050 flinkname = "";
6051 fbasedir = os.path.dirname(fname);
6052 fcurfid = curfid;
6053 fcurinode = curfid;
6054 finode = fcurinode;
6055 curfid = curfid + 1;
6056 fdev_minor = 0;
6057 fdev_major = 0;
6058 frdev_minor = 0;
6059 frdev_major = 0;
6060 if(ftype==5):
6061 fsize = "0";
6062 fatime = int(member.creationtime.timestamp());
6063 fmtime = int(member.creationtime.timestamp());
6064 fctime = int(member.creationtime.timestamp());
6065 fbtime = int(member.creationtime.timestamp());
6066 if(member.is_directory):
6067 fmode = int(stat.S_IFDIR + 511)
6068 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6069 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6070 else:
6071 fmode = int(stat.S_IFLNK + 438)
6072 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6073 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6074 try:
6075 fuid = os.getuid();
6076 except AttributeError:
6077 fuid = 0;
6078 except KeyError:
6079 fuid = 0;
6080 try:
6081 fgid = os.getgid();
6082 except AttributeError:
6083 fgid = 0;
6084 except KeyError:
6085 fgid = 0;
6086 try:
6087 import pwd;
6088 try:
6089 userinfo = pwd.getpwuid(os.getuid());
6090 funame = userinfo.pw_name;
6091 except KeyError:
6092 funame = "";
6093 except AttributeError:
6094 funame = "";
6095 except ImportError:
6096 funame = "";
6097 fgname = "";
6098 try:
6099 import grp;
6100 try:
6101 groupinfo = grp.getgrgid(os.getgid());
6102 fgname = groupinfo.gr_name;
6103 except KeyError:
6104 fgname = "";
6105 except AttributeError:
6106 fgname = "";
6107 except ImportError:
6108 fgname = "";
6109 fcontents = BytesIO();
6110 if(ftype==0):
6111 fcontents.write(file_content[member.filename].read());
6112 fsize = format(fcontents.tell(), 'x').lower();
6113 fileop.close();
6114 fcontents.seek(0, 0);
6115 ftypehex = format(ftype, 'x').lower();
6116 extrafields = len(extradata);
6117 extrafieldslist = extradata;
6118 catfextrafields = extrafields;
6119 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6120 if(len(extradata)>0):
6121 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6122 extrasizelen = len(extrasizestr);
6123 extrasizelenhex = format(extrasizelen, 'x').lower();
6124 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6125 catoutlen = len(catoutlist) + len(extradata) + 3;
6126 catoutlenhex = format(catoutlen, 'x').lower();
6127 catoutlist.insert(0, catoutlenhex);
6128 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6129 catheaderdata = catoutlist;
6130 if(len(extradata)>0):
6131 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6132 if(fsize==0):
6133 checksumlist = [checksumtype, "none"];
6134 else:
6135 checksumlist = [checksumtype, checksumtype];
6136 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6137 catfnumfields = 24 + catfextrafields;
6138 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6139 fcontents.seek(0, 0);
6140 if(fsize==0):
6141 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6142 else:
6143 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6144 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6145 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6146 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6147 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6148 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6149 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6150 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6151 catfcontentstart = fheadtell;
6152 fheadtell += len(catfileoutstr) + 1;
6153 catfcontentend = fheadtell - 1;
6154 catfhend = catfcontentend;
6155 fcontents.seek(0, 0);
6156 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6157 pyhascontents = False;
6158 if(int(fsize)>0 and not listonly):
6159 pyhascontents = True;
6160 if(int(fsize)>0 and listonly):
6161 fcontents = BytesIO();
6162 pyhascontents = False;
6163 fcontents.seek(0, 0);
6164 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6165 fileidnum = fileidnum + 1;
6166 return catlist;
6167
6168 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6169 formatspecs = FormatSpecsListToDict(formatspecs);
6170 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6171 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
6172 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6173 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
6174 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6175 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
6176 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6177 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
6178 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6179 elif(checkcompressfile=="catfile"):
6180 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6181 else:
6182 return False;
6183 return False;
6184
6185 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6186 formatspecs = FormatSpecsListToDict(formatspecs);
6187 outarray = BytesIO();
6188 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6189 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6190 return listcatfiles;
6191
6192 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6193 formatspecs = FormatSpecsListToDict(formatspecs);
6194 if(isinstance(inarray, dict)):
6195 listcatfiles = inarray;
6196 else:
6197 return False;
6198 if(not listcatfiles):
6199 return False;
6200 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6201 if(returnfp):
6202 catarray.update({'catfp': listcatfiles['catfp']});
6203 lenlist = len(listcatfiles['ffilelist']);
6204 lcfi = 0;
6205 lcfx = int(listcatfiles['fnumfiles']);
6206 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6207 lcfx = int(lenlist);
6208 else:
6209 lcfx = int(listcatfiles['fnumfiles']);
6210 while(lcfi < lcfx):
6211 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6212 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6213 catarray['filetoid'].update(filetoidarray);
6214 catarray['idtofile'].update(idtofilearray);
6215 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6216 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6217 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6218 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6219 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6220 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6221 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6222 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6223 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6224 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6225 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6226 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6227 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6228 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6229 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6230 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6231 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6232 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6233 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6234 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6235 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6236 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6237 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6238 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6239 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6240 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6241 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6242 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6243 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6244 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6245 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6246 lcfi = lcfi + 1;
6247 return catarray;
6248
6249 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6250
6251 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6252 formatspecs = FormatSpecsListToDict(formatspecs);
6253 if(isinstance(infile, dict)):
6254 listcatfiles = infile;
6255 else:
6256 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6257 infile = RemoveWindowsPath(infile);
6258 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6259 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6260 outfile = RemoveWindowsPath(outfile);
6261 checksumtype = checksumtype.lower();
6262 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6263 checksumtype="crc32";
6264 if(checksumtype=="none"):
6265 checksumtype = "";
6266 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6267 compression = "auto";
6268 if(compression not in compressionlist and compression is None):
6269 compression = "auto";
6270 if(verbose):
6271 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6272 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6273 if(os.path.exists(outfile)):
6274 try:
6275 os.unlink(outfile);
6276 except OSError as e:
6277 pass;
6278 if(not listcatfiles):
6279 return False;
6280 if(outfile=="-"):
6281 verbose = False;
6282 catfp = BytesIO();
6283 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6284 catfp = outfile;
6285 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6286 catfp = BytesIO();
6287 else:
6288 fbasename = os.path.splitext(outfile)[0];
6289 fextname = os.path.splitext(outfile)[1];
6290 if(not compresswholefile and fextname in outextlistwd):
6291 compresswholefile = True;
6292 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6293 catver = formatspecs['format_ver'];
6294 fileheaderver = str(int(catver.replace(".", "")));
6295 lenlist = len(listcatfiles['ffilelist']);
6296 fnumfiles = int(listcatfiles['fnumfiles']);
6297 if(lenlist>fnumfiles or lenlist<fnumfiles):
6298 fnumfiles = lenlist;
6299 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6300 lenlist = len(listcatfiles['ffilelist']);
6301 fnumfiles = int(listcatfiles['fnumfiles']);
6302 lcfi = 0;
6303 lcfx = int(listcatfiles['fnumfiles']);
6304 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6305 lcfx = int(lenlist);
6306 else:
6307 lcfx = int(listcatfiles['fnumfiles']);
6308 curinode = 0;
6309 curfid = 0;
6310 inodelist = [];
6311 inodetofile = {};
6312 filetoinode = {};
6313 reallcfi = 0;
6314 while(lcfi < lcfx):
6315 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6316 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6317 else:
6318 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6319 if(verbose):
6320 VerbosePrintOut(fname);
6321 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6322 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6323 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6324 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6325 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6326 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6327 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6328 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6329 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6330 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6331 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6332 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6333 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6334 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6335 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6336 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6337 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6338 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6339 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6340 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6341 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6342 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6343 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6344 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6345 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6346 if(not followlink and len(extradata)<0):
6347 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6348 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6349 fcompression = "";
6350 fcsize = format(int(0), 'x').lower();
6351 if(not compresswholefile):
6352 fcontents.seek(0, 2);
6353 ucfsize = fcontents.tell();
6354 fcontents.seek(0, 0);
6355 if(compression=="auto"):
6356 ilsize = len(compressionlistalt);
6357 ilmin = 0;
6358 ilcsize = [];
6359 while(ilmin < ilsize):
6360 cfcontents = BytesIO();
6361 shutil.copyfileobj(fcontents, cfcontents);
6362 fcontents.seek(0, 0);
6363 cfcontents.seek(0, 0);
6364 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6365 if(cfcontents):
6366 cfcontents.seek(0, 2);
6367 ilcsize.append(cfcontents.tell());
6368 cfcontents.close();
6369 else:
6370 try:
6371 ilcsize.append(sys.maxint);
6372 except AttributeError:
6373 ilcsize.append(sys.maxsize);
6374 ilmin = ilmin + 1;
6375 ilcmin = ilcsize.index(min(ilcsize));
6376 compression = compressionlistalt[ilcmin];
6377 fcontents.seek(0, 0);
6378 cfcontents = BytesIO();
6379 shutil.copyfileobj(fcontents, cfcontents);
6380 cfcontents.seek(0, 0);
6381 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6382 cfcontents.seek(0, 2);
6383 cfsize = cfcontents.tell();
6384 if(ucfsize > cfsize):
6385 fcsize = format(int(cfsize), 'x').lower();
6386 fcompression = compression;
6387 fcontents.close();
6388 fcontents = cfcontents;
6389 if(followlink):
6390 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6391 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6392 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6393 flinkinfo = listcatfiles['ffilelist'][flinkid];
6394 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6395 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6396 flinkname = flinkinfo['flinkname'];
6397 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6398 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6399 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6400 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6401 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6402 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6403 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6404 funame = flinkinfo['funame'];
6405 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6406 fgname = flinkinfo['fgname'];
6407 finode = format(int(flinkinfo['finode']), 'x').lower();
6408 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6409 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6410 fcompression = flinkinfo['fcompression'];
6411 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6412 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6413 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6414 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6415 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6416 fseeknextfile = flinkinfo['fseeknextfile'];
6417 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6418 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6419 if(len(extradata)<0):
6420 extradata = flinkinfo['fextralist'];
6421 fcontents = flinkinfo['fcontents'];
6422 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6423 else:
6424 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6425 fcurfid = format(curfid, 'x').lower();
6426 if(not followlink and finode!=0):
6427 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6428 fcurinode = format(int(curinode), 'x').lower();
6429 inodetofile.update({curinode: fname});
6430 filetoinode.update({fname: curinode});
6431 curinode = curinode + 1;
6432 else:
6433 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6434 else:
6435 fcurinode = format(int(curinode), 'x').lower();
6436 curinode = curinode + 1;
6437 curfid = curfid + 1;
6438 if(fcompression=="none"):
6439 fcompression = "";
6440 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6441 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6442 fcontents.close();
6443 lcfi = lcfi + 1;
6444 reallcfi = reallcfi + 1;
6445 if(lcfx>0):
6446 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6447 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6448 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6449 try:
6450 catfp.flush();
6451 os.fsync(catfp.fileno());
6452 except io.UnsupportedOperation:
6453 pass;
6454 except AttributeError:
6455 pass;
6456 except OSError as e:
6457 pass;
6458 if(outfile=="-"):
6459 catfp.seek(0, 0);
6460 if(hasattr(sys.stdout, "buffer")):
6461 shutil.copyfileobj(catfp, sys.stdout.buffer);
6462 else:
6463 shutil.copyfileobj(catfp, sys.stdout);
6464 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6465 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6466 catfp.seek(0, 0);
6467 upload_file_to_internet_file(catfp, outfile);
6468 if(returnfp):
6469 catfp.seek(0, 0);
6470 return catfp;
6471 else:
6472 catfp.close();
6473 return True;
6474
6475 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6476
6477 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6478 formatspecs = FormatSpecsListToDict(formatspecs);
6479 catfp = BytesIO(catstr);
6480 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6481 return listcatfiles;
6482
6483 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6484
6485 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6486 formatspecs = FormatSpecsListToDict(formatspecs);
6487 outarray = BytesIO();
6488 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6489 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6490 return listcatfiles;
6491
6492 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6493
6494 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6495 formatspecs = FormatSpecsListToDict(formatspecs);
6496 if(outdir is not None):
6497 outdir = RemoveWindowsPath(outdir);
6498 if(verbose):
6499 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6500 if(isinstance(infile, dict)):
6501 listcatfiles = infile;
6502 else:
6503 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6504 infile = RemoveWindowsPath(infile);
6505 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6506 if(not listcatfiles):
6507 return False;
6508 lenlist = len(listcatfiles['ffilelist']);
6509 fnumfiles = int(listcatfiles['fnumfiles']);
6510 lcfi = 0;
6511 lcfx = int(listcatfiles['fnumfiles']);
6512 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6513 lcfx = int(lenlist);
6514 else:
6515 lcfx = int(listcatfiles['fnumfiles']);
6516 while(lcfi<lcfx):
6517 funame = "";
6518 try:
6519 import pwd;
6520 try:
6521 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6522 funame = userinfo.pw_name;
6523 except KeyError:
6524 funame = "";
6525 except ImportError:
6526 funame = "";
6527 fgname = "";
6528 try:
6529 import grp;
6530 try:
6531 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6532 fgname = groupinfo.gr_name;
6533 except KeyError:
6534 fgname = "";
6535 except ImportError:
6536 fgname = "";
6537 if(verbose):
6538 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6539 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6540 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6541 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6542 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6543 try:
6544 fpc.flush();
6545 os.fsync(fpc.fileno());
6546 except io.UnsupportedOperation:
6547 pass;
6548 except AttributeError:
6549 pass;
6550 except OSError as e:
6551 pass;
6552 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6553 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6554 if(preservepermissions):
6555 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6556 if(preservetime):
6557 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6558 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6559 if(followlink):
6560 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6561 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6562 flinkinfo = listcatfiles['ffilelist'][flinkid];
6563 funame = "";
6564 try:
6565 import pwd;
6566 try:
6567 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6568 funame = userinfo.pw_name;
6569 except KeyError:
6570 funame = "";
6571 except ImportError:
6572 funame = "";
6573 fgname = "";
6574 try:
6575 import grp;
6576 try:
6577 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6578 fgname = groupinfo.gr_name;
6579 except KeyError:
6580 fgname = "";
6581 except ImportError:
6582 fgname = "";
6583 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6584 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6585 flinkinfo['fcontents'].seek(0, 0);
6586 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6587 try:
6588 fpc.flush();
6589 os.fsync(fpc.fileno());
6590 except io.UnsupportedOperation:
6591 pass;
6592 except AttributeError:
6593 pass;
6594 except OSError as e:
6595 pass;
6596 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6597 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6598 if(preservepermissions):
6599 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6600 if(preservetime):
6601 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6602 if(flinkinfo['ftype']==1):
6603 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6604 if(flinkinfo['ftype']==2):
6605 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6606 if(flinkinfo['ftype']==5):
6607 if(preservepermissions):
6608 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6609 else:
6610 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6611 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6612 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6613 if(preservepermissions):
6614 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6615 if(preservetime):
6616 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6617 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6618 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6619 else:
6620 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6621 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6622 if(followlink):
6623 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6624 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6625 flinkinfo = listcatfiles['ffilelist'][flinkid];
6626 funame = "";
6627 try:
6628 import pwd;
6629 try:
6630 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6631 funame = userinfo.pw_name;
6632 except KeyError:
6633 funame = "";
6634 except ImportError:
6635 funame = "";
6636 fgname = "";
6637 try:
6638 import grp;
6639 try:
6640 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6641 fgname = groupinfo.gr_name;
6642 except KeyError:
6643 fgname = "";
6644 except ImportError:
6645 fgname = "";
6646 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6647 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6648 flinkinfo['fcontents'].seek(0, 0);
6649 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6650 try:
6651 fpc.flush();
6652 os.fsync(fpc.fileno());
6653 except io.UnsupportedOperation:
6654 pass;
6655 except AttributeError:
6656 pass;
6657 except OSError as e:
6658 pass;
6659 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6660 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6661 if(preservepermissions):
6662 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6663 if(preservetime):
6664 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6665 if(flinkinfo['ftype']==1):
6666 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6667 if(flinkinfo['ftype']==2):
6668 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6669 if(flinkinfo['ftype']==5):
6670 if(preservepermissions):
6671 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6672 else:
6673 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6674 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6675 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6676 if(preservepermissions):
6677 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6678 if(preservetime):
6679 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6680 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6681 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6682 else:
6683 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6684 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6685 if(preservepermissions):
6686 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6687 else:
6688 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6689 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6690 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6691 if(preservepermissions):
6692 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6693 if(preservetime):
6694 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6695 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6696 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6697 lcfi = lcfi + 1;
6698 if(returnfp):
6699 return listcatfiles['ffilelist']['catfp'];
6700 else:
6701 return True;
6702
6703 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6704
6705 if(hasattr(shutil, "register_unpack_format")):
6706 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6707 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6708 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6709
6710 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6711 formatspecs = FormatSpecsListToDict(formatspecs);
6712 catfp = BytesIO(catstr);
6713 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6714 return listcatfiles;
6715
6716 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6717
6718 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6719 formatspecs = FormatSpecsListToDict(formatspecs);
6720 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6721 if(isinstance(infile, dict)):
6722 listcatfiles = infile;
6723 else:
6724 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6725 infile = RemoveWindowsPath(infile);
6726 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6727 if(not listcatfiles):
6728 return False;
6729 lenlist = len(listcatfiles['ffilelist']);
6730 fnumfiles = int(listcatfiles['fnumfiles']);
6731 lcfi = 0;
6732 lcfx = int(listcatfiles['fnumfiles']);
6733 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6734 lcfx = int(lenlist);
6735 else:
6736 lcfx = int(listcatfiles['fnumfiles']);
6737 returnval = {};
6738 while(lcfi<lcfx):
6739 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6740 if(not verbose):
6741 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6742 if(verbose):
6743 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6744 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6745 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6746 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6747 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6748 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6749 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6750 if(len(fuprint)<=0):
6751 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6752 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6753 if(len(fgprint)<=0):
6754 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6755 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6756 lcfi = lcfi + 1;
6757 if(returnfp):
6758 return listcatfiles['catfp'];
6759 else:
6760 return True;
6761
6762 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6763
6764 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6765 formatspecs = FormatSpecsListToDict(formatspecs);
6766 catfp = BytesIO(catstr);
6767 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6768 return listcatfiles;
6769
6770 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6771
6772 def TarFileListFiles(infile, verbose=False, returnfp=False):
6773 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6774 if(infile=="-"):
6775 infile = BytesIO();
6776 if(hasattr(sys.stdin, "buffer")):
6777 shutil.copyfileobj(sys.stdin.buffer, infile);
6778 else:
6779 shutil.copyfileobj(sys.stdin, infile);
6780 infile.seek(0, 0);
6781 if(not infile):
6782 return False;
6783 infile.seek(0, 0);
6784 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6785 infile = download_file_from_internet_file(infile);
6786 infile.seek(0, 0);
6787 if(not infile):
6788 return False;
6789 infile.seek(0, 0);
6790 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6791 return False;
6792 elif(os.path.exists(infile) and os.path.isfile(infile)):
6793 try:
6794 if(not tarfile.TarFileCheck(infile)):
6795 return False;
6796 except AttributeError:
6797 if(not TarFileCheck(infile)):
6798 return False;
6799 try:
6800 if(hasattr(infile, "read") or hasattr(infile, "write")):
6801 tarfp = tarfile.open(fileobj=infile, mode="r");
6802 else:
6803 tarfp = tarfile.open(infile, "r");
6804 except FileNotFoundError:
6805 return False;
6806 lcfi = 0
6807 returnval = {};
6808 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6809 returnval.update({lcfi: member.name});
6810 fpremode = member.mode;
6811 ffullmode = member.mode;
6812 flinkcount = 0;
6813 ftype = 0;
6814 if(member.isreg()):
6815 ffullmode = member.mode + stat.S_IFREG;
6816 ftype = 0;
6817 elif(member.isdev()):
6818 ffullmode = member.mode;
6819 ftype = 7;
6820 elif(member.islnk()):
6821 ffullmode = member.mode + stat.S_IFREG;
6822 ftype = 1;
6823 elif(member.issym()):
6824 ffullmode = member.mode + stat.S_IFLNK;
6825 ftype = 2;
6826 elif(member.ischr()):
6827 ffullmode = member.mode + stat.S_IFCHR;
6828 ftype = 3;
6829 elif(member.isblk()):
6830 ffullmode = member.mode + stat.S_IFBLK;
6831 ftype = 4;
6832 elif(member.isdir()):
6833 ffullmode = member.mode + stat.S_IFDIR;
6834 ftype = 5;
6835 elif(member.isfifo()):
6836 ffullmode = member.mode + stat.S_IFIFO;
6837 ftype = 6;
6838 elif(member.issparse()):
6839 ffullmode = member.mode;
6840 ftype = 12;
6841 if(not verbose):
6842 VerbosePrintOut(member.name);
6843 elif(verbose):
6844 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6845 printfname = member.name;
6846 if(member.islnk()):
6847 printfname = member.name + " link to " + member.linkname;
6848 elif(member.issym()):
6849 printfname = member.name + " -> " + member.linkname;
6850 fuprint = member.uname;
6851 if(len(fuprint)<=0):
6852 fuprint = member.uid;
6853 fgprint = member.gname;
6854 if(len(fgprint)<=0):
6855 fgprint = member.gid;
6856 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6857 lcfi = lcfi + 1;
6858 if(returnfp):
6859 return listcatfiles['catfp'];
6860 else:
6861 return True;
6862
6863 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6864 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6865 if(infile=="-"):
6866 infile = BytesIO();
6867 if(hasattr(sys.stdin, "buffer")):
6868 shutil.copyfileobj(sys.stdin.buffer, infile);
6869 else:
6870 shutil.copyfileobj(sys.stdin, infile);
6871 infile.seek(0, 0);
6872 if(not infile):
6873 return False;
6874 infile.seek(0, 0);
6875 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6876 infile = download_file_from_internet_file(infile);
6877 infile.seek(0, 0);
6878 if(not infile):
6879 return False;
6880 infile.seek(0, 0);
6881 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6882 return False;
6883 if(not zipfile.is_zipfile(infile)):
6884 return False;
6885 try:
6886 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6887 except FileNotFoundError:
6888 print(6);return False;
6889 lcfi = 0;
6890 returnval = {};
6891 ziptest = zipfp.testzip();
6892 if(ziptest):
6893 VerbosePrintOut("Bad file found!");
6894 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6895 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6896 fwinattributes = int(zipinfo.external_attr);
6897 if(not member.is_dir()):
6898 fmode = int(stat.S_IFREG + 438);
6899 fchmode = int(stat.S_IMODE(fmode));
6900 ftypemod = int(stat.S_IFMT(fmode));
6901 elif(member.is_dir()):
6902 fmode = int(stat.S_IFDIR + 511);
6903 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6904 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6905 elif(zipinfo.create_system==3):
6906 fwinattributes =int(0);
6907 fmode = int(zipinfo.external_attr);
6908 fchmode = int(stat.S_IMODE(fmode));
6909 ftypemod = int(stat.S_IFMT(fmode));
6910 else:
6911 fwinattributes = int(0);
6912 if(not member.is_dir()):
6913 fmode = int(stat.S_IFREG + 438);
6914 fchmode = int(stat.S_IMODE(fmode));
6915 ftypemod = int(stat.S_IFMT(fmode));
6916 elif(member.is_dir()):
6917 fmode = int(stat.S_IFDIR + 511);
6918 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6919 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6920 returnval.update({lcfi: member.filename});
6921 if(not verbose):
6922 VerbosePrintOut(member.filename);
6923 if(verbose):
6924 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6925 permissionstr = "";
6926 for fmodval in str(oct(fmode))[-3:]:
6927 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6928 if(not member.is_dir()):
6929 ftype = 0;
6930 permissionstr = "-" + permissionstr;
6931 elif(member.is_dir()):
6932 ftype = 5;
6933 permissionstr = "d" + permissionstr;
6934 printfname = member.filename;
6935 try:
6936 fuid = int(os.getuid());
6937 except AttributeError:
6938 fuid = int(0);
6939 except KeyError:
6940 fuid = int(0);
6941 try:
6942 fgid = int(os.getgid());
6943 except AttributeError:
6944 fgid = int(0);
6945 except KeyError:
6946 fgid = int(0);
6947 try:
6948 import pwd;
6949 try:
6950 userinfo = pwd.getpwuid(os.getuid());
6951 funame = userinfo.pw_name;
6952 except KeyError:
6953 funame = "";
6954 except AttributeError:
6955 funame = "";
6956 except ImportError:
6957 funame = "";
6958 fgname = "";
6959 try:
6960 import grp;
6961 try:
6962 groupinfo = grp.getgrgid(os.getgid());
6963 fgname = groupinfo.gr_name;
6964 except KeyError:
6965 fgname = "";
6966 except AttributeError:
6967 fgname = "";
6968 except ImportError:
6969 fgname = "";
6970 fuprint = funame;
6971 if(len(fuprint)<=0):
6972 fuprint = str(fuid);
6973 fgprint = fgname;
6974 if(len(fgprint)<=0):
6975 fgprint = str(fgid);
6976 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6977 lcfi = lcfi + 1;
6978 if(returnfp):
6979 return listcatfiles['catfp'];
6980 else:
6981 return True;
6982
6983 if(not rarfile_support):
6984 def RarFileListFiles(infile, verbose=False, returnfp=False):
6985 return False;
6986
6987 if(rarfile_support):
6988 def RarFileListFiles(infile, verbose=False, returnfp=False):
6989 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6990 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6991 return False;
6992 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6993 return False;
6994 lcfi = 0;
6995 returnval = {};
6996 rarfp = rarfile.RarFile(infile, "r");
6997 rartest = rarfp.testrar();
6998 if(rartest):
6999 VerbosePrintOut("Bad file found!");
7000 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7001 is_unix = False;
7002 is_windows = False;
7003 if(member.host_os==rarfile.RAR_OS_UNIX):
7004 is_windows = False;
7005 try:
7006 member.external_attr
7007 is_unix = True;
7008 except AttributeError:
7009 is_unix = False;
7010 elif(member.host_os==rarfile.RAR_OS_WIN32):
7011 is_unix = False;
7012 try:
7013 member.external_attr
7014 is_windows = True;
7015 except AttributeError:
7016 is_windows = False;
7017 else:
7018 is_unix = False;
7019 is_windows = False;
7020 if(is_unix and member.external_attr !=0):
7021 fpremode = int(member.external_attr);
7022 elif(member.is_file()):
7023 fpremode = int(stat.S_IFREG + 438);
7024 elif(member.is_symlink()):
7025 fpremode = int(stat.S_IFLNK + 438);
7026 elif(member.is_dir()):
7027 fpremode = int(stat.S_IFDIR + 511);
7028 if(is_windows and member.external_attr !=0):
7029 fwinattributes = int(member.external_attr);
7030 else:
7031 fwinattributes = int(0);
7032 if(is_unix and member.external_attr !=0):
7033 fmode = int(member.external_attr);
7034 fchmode = int(stat.S_IMODE(member.external_attr));
7035 ftypemod = int(stat.S_IFMT(member.external_attr));
7036 elif(member.is_file()):
7037 fmode = int(stat.S_IFREG + 438);
7038 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7039 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7040 elif(member.is_symlink()):
7041 fmode = int(stat.S_IFLNK + 438);
7042 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7043 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7044 elif(member.is_dir()):
7045 fmode = int(stat.S_IFDIR + 511);
7046 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7047 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7048 returnval.update({lcfi: member.filename});
7049 if(not verbose):
7050 VerbosePrintOut(member.filename);
7051 if(verbose):
7052 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7053 permissionstr = "";
7054 for fmodval in str(oct(fmode))[-3:]:
7055 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7056 if(member.is_file()):
7057 ftype = 0;
7058 permissionstr = "-" + permissionstr;
7059 printfname = member.filename;
7060 elif(member.is_symlink()):
7061 ftype = 2;
7062 permissionstr = "l" + permissionstr;
7063 printfname = member.name + " -> " + member.read().decode("UTF-8");
7064 elif(member.is_dir()):
7065 ftype = 5;
7066 permissionstr = "d" + permissionstr;
7067 printfname = member.filename;
7068 try:
7069 fuid = int(os.getuid());
7070 except AttributeError:
7071 fuid = int(0);
7072 except KeyError:
7073 fuid = int(0);
7074 try:
7075 fgid = int(os.getgid());
7076 except AttributeError:
7077 fgid = int(0);
7078 except KeyError:
7079 fgid = int(0);
7080 try:
7081 import pwd;
7082 try:
7083 userinfo = pwd.getpwuid(os.getuid());
7084 funame = userinfo.pw_name;
7085 except KeyError:
7086 funame = "";
7087 except AttributeError:
7088 funame = "";
7089 except ImportError:
7090 funame = "";
7091 fgname = "";
7092 try:
7093 import grp;
7094 try:
7095 groupinfo = grp.getgrgid(os.getgid());
7096 fgname = groupinfo.gr_name;
7097 except KeyError:
7098 fgname = "";
7099 except AttributeError:
7100 fgname = "";
7101 except ImportError:
7102 fgname = "";
7103 fuprint = funame;
7104 if(len(fuprint)<=0):
7105 fuprint = str(fuid);
7106 fgprint = fgname;
7107 if(len(fgprint)<=0):
7108 fgprint = str(fgid);
7109 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7110 lcfi = lcfi + 1;
7111 if(returnfp):
7112 return listcatfiles['catfp'];
7113 else:
7114 return True;
7115
7116 if(not py7zr_support):
7117 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7118 return False;
7119
7120 if(py7zr_support):
7121 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7122 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7123 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7124 return False;
7125 lcfi = 0;
7126 returnval = {};
7127 szpfp = py7zr.SevenZipFile(infile, mode="r");
7128 file_content = szpfp.readall();
7129 #sztest = szpfp.testzip();
7130 sztestalt = szpfp.test();
7131 if(sztestalt):
7132 VerbosePrintOut("Bad file found!");
7133 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7134 if(re.findall(r"^[.|/]", member.filename)):
7135 fname = member.filename;
7136 else:
7137 fname = "./"+member.filename;
7138 if(not member.is_directory):
7139 fpremode = int(stat.S_IFREG + 438);
7140 elif(member.is_directory):
7141 fpremode = int(stat.S_IFDIR + 511);
7142 fwinattributes = int(0);
7143 if(member.is_directory):
7144 fmode = int(stat.S_IFDIR + 511);
7145 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7146 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7147 else:
7148 fmode = int(stat.S_IFLNK + 438);
7149 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7150 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7151 returnval.update({lcfi: member.filename});
7152 if(not verbose):
7153 VerbosePrintOut(member.filename);
7154 if(verbose):
7155 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7156 permissionstr = "";
7157 for fmodval in str(oct(fmode))[-3:]:
7158 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7159 fsize = int("0");
7160 if(not member.is_directory):
7161 ftype = 0;
7162 permissionstr = "-" + permissionstr;
7163 printfname = member.filename;
7164 elif(member.is_directory):
7165 ftype = 5;
7166 permissionstr = "d" + permissionstr;
7167 printfname = member.filename;
7168 if(ftype==0):
7169 fsize = len(file_content[member.filename].read());
7170 file_content[member.filename].close();
7171 try:
7172 fuid = int(os.getuid());
7173 except AttributeError:
7174 fuid = int(0);
7175 except KeyError:
7176 fuid = int(0);
7177 try:
7178 fgid = int(os.getgid());
7179 except AttributeError:
7180 fgid = int(0);
7181 except KeyError:
7182 fgid = int(0);
7183 try:
7184 import pwd;
7185 try:
7186 userinfo = pwd.getpwuid(os.getuid());
7187 funame = userinfo.pw_name;
7188 except KeyError:
7189 funame = "";
7190 except AttributeError:
7191 funame = "";
7192 except ImportError:
7193 funame = "";
7194 fgname = "";
7195 try:
7196 import grp;
7197 try:
7198 groupinfo = grp.getgrgid(os.getgid());
7199 fgname = groupinfo.gr_name;
7200 except KeyError:
7201 fgname = "";
7202 except AttributeError:
7203 fgname = "";
7204 except ImportError:
7205 fgname = "";
7206 fuprint = funame;
7207 if(len(fuprint)<=0):
7208 fuprint = str(fuid);
7209 fgprint = fgname;
7210 if(len(fgprint)<=0):
7211 fgprint = str(fgid);
7212 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7213 lcfi = lcfi + 1;
7214 if(returnfp):
7215 return listcatfiles['catfp'];
7216 else:
7217 return True;
7218
7219 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7220 formatspecs = FormatSpecsListToDict(formatspecs);
7221 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7222 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7223 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
7224 return TarFileListFiles(infile, verbose, returnfp);
7225 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
7226 return ZipFileListFiles(infile, verbose, returnfp);
7227 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
7228 return RarFileListFiles(infile, verbose, returnfp);
7229 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
7230 return SevenZipFileListFiles(infile, verbose, returnfp);
7231 elif(checkcompressfile=="catfile"):
7232 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7233 else:
7234 return False;
7235 return False;
7236
7237 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7238 formatspecs = FormatSpecsListToDict(formatspecs);
7239 outarray = BytesIO();
7240 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7241 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7242 return listcatfiles;
7243
7244 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7245 formatspecs = FormatSpecsListToDict(formatspecs);
7246 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7247 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7248 return listcatfiles;
7249
7250 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7251 formatspecs = FormatSpecsListToDict(formatspecs);
7252 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7253 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7254 return listcatfiles;
7255
7256 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7257
7258 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7259 formatspecs = FormatSpecsListToDict(formatspecs);
7260 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7261 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7262 return listcatfiles;
7263
7264 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7265
7266 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7267 formatspecs = FormatSpecsListToDict(formatspecs);
7268 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7269 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7270 return listcatfiles;
7271
7272 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7273
7274 if(not rarfile_support):
7275 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7276 return False;
7277
7278 if(rarfile_support):
7279 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7280 formatspecs = FormatSpecsListToDict(formatspecs);
7281 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7282 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7283 return listcatfiles;
7284
7285 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7286
7287 if(not py7zr_support):
7288 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7289 return False;
7290
7291 if(py7zr_support):
7292 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7293 formatspecs = FormatSpecsListToDict(formatspecs);
7294 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7295 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7296 return listcatfiles;
7297
7298 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7299
7300 def download_file_from_ftp_file(url):
7301 urlparts = urlparse(url);
7302 file_name = os.path.basename(urlparts.path);
7303 file_dir = os.path.dirname(urlparts.path);
7304 if(urlparts.username is not None):
7305 ftp_username = urlparts.username;
7306 else:
7307 ftp_username = "anonymous";
7308 if(urlparts.password is not None):
7309 ftp_password = urlparts.password;
7310 elif(urlparts.password is None and urlparts.username=="anonymous"):
7311 ftp_password = "anonymous";
7312 else:
7313 ftp_password = "";
7314 if(urlparts.scheme=="ftp"):
7315 ftp = FTP();
7316 elif(urlparts.scheme=="ftps" and ftpssl):
7317 ftp = FTP_TLS();
7318 else:
7319 return False;
7320 if(urlparts.scheme=="sftp"):
7321 if(__use_pysftp__):
7322 return download_file_from_pysftp_file(url);
7323 else:
7324 return download_file_from_sftp_file(url);
7325 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7326 return download_file_from_http_file(url);
7327 ftp_port = urlparts.port;
7328 if(urlparts.port is None):
7329 ftp_port = 21;
7330 try:
7331 ftp.connect(urlparts.hostname, ftp_port);
7332 except socket.gaierror:
7333 log.info("Error With URL "+url);
7334 return False;
7335 except socket.timeout:
7336 log.info("Error With URL "+url);
7337 return False;
7338 ftp.login(urlparts.username, urlparts.password);
7339 if(urlparts.scheme=="ftps"):
7340 ftp.prot_p();
7341 ftpfile = BytesIO();
7342 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7343 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7344 ftp.close();
7345 ftpfile.seek(0, 0);
7346 return ftpfile;
7347
7348 def download_file_from_ftp_string(url):
7349 ftpfile = download_file_from_ftp_file(url);
7350 return ftpfile.read();
7351
7352 def upload_file_to_ftp_file(ftpfile, url):
7353 urlparts = urlparse(url);
7354 file_name = os.path.basename(urlparts.path);
7355 file_dir = os.path.dirname(urlparts.path);
7356 if(urlparts.username is not None):
7357 ftp_username = urlparts.username;
7358 else:
7359 ftp_username = "anonymous";
7360 if(urlparts.password is not None):
7361 ftp_password = urlparts.password;
7362 elif(urlparts.password is None and urlparts.username=="anonymous"):
7363 ftp_password = "anonymous";
7364 else:
7365 ftp_password = "";
7366 if(urlparts.scheme=="ftp"):
7367 ftp = FTP();
7368 elif(urlparts.scheme=="ftps" and ftpssl):
7369 ftp = FTP_TLS();
7370 else:
7371 return False;
7372 if(urlparts.scheme=="sftp"):
7373 if(__use_pysftp__):
7374 return upload_file_to_pysftp_file(url);
7375 else:
7376 return upload_file_to_sftp_file(url);
7377 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7378 return False;
7379 ftp_port = urlparts.port;
7380 if(urlparts.port is None):
7381 ftp_port = 21;
7382 try:
7383 ftp.connect(urlparts.hostname, ftp_port);
7384 except socket.gaierror:
7385 log.info("Error With URL "+url);
7386 return False;
7387 except socket.timeout:
7388 log.info("Error With URL "+url);
7389 return False;
7390 ftp.login(urlparts.username, urlparts.password);
7391 if(urlparts.scheme=="ftps"):
7392 ftp.prot_p();
7393 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7394 ftp.close();
7395 ftpfile.seek(0, 0);
7396 return ftpfile;
7397
7398 def upload_file_to_ftp_string(ftpstring, url):
7399 ftpfileo = BytesIO(ftpstring);
7400 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7401 ftpfileo.close();
7402 return ftpfile;
7403
7404 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7405 # Parse the URL to extract username and password if present
7406 urlparts = urlparse(url);
7407 username = urlparts.username;
7408 password = urlparts.password;
7409 # Rebuild the URL without the username and password
7410 netloc = urlparts.hostname;
7411 if(urlparts.scheme=="sftp"):
7412 if(__use_pysftp__):
7413 return download_file_from_pysftp_file(url);
7414 else:
7415 return download_file_from_sftp_file(url);
7416 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7417 return download_file_from_ftp_file(url);
7418 if urlparts.port:
7419 netloc += ':' + str(urlparts.port);
7420 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7421 # Create a temporary file object
7422 httpfile = BytesIO();
7423 if haverequests:
7424 # Use the requests library if available
7425 if username and password:
7426 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7427 else:
7428 response = requests.get(rebuilt_url, headers=headers, stream=True);
7429 response.raw.decode_content = True
7430 shutil.copyfileobj(response.raw, httpfile);
7431 else:
7432 # Build a Request object for urllib
7433 request = Request(rebuilt_url, headers=headers);
7434 # Create an opener object for handling URLs
7435 if username and password:
7436 # Create a password manager
7437 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7438 # Add the username and password
7439 password_mgr.add_password(None, rebuilt_url, username, password);
7440 # Create an authentication handler using the password manager
7441 auth_handler = HTTPBasicAuthHandler(password_mgr);
7442 # Build the opener with the authentication handler
7443 opener = build_opener(auth_handler);
7444 else:
7445 opener = build_opener();
7446 with opener.open(request) as response:
7447 shutil.copyfileobj(response, httpfile);
7448 # Reset file pointer to the start
7449 httpfile.seek(0, 0);
7450 # Return the temporary file object
7451 return httpfile;
7452
7453 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7454 httpfile = download_file_from_http_file(url, headers);
7455 return ftpfile.read();
7456
7457 if(haveparamiko):
7458 def download_file_from_sftp_file(url):
7459 urlparts = urlparse(url);
7460 file_name = os.path.basename(urlparts.path);
7461 file_dir = os.path.dirname(urlparts.path);
7462 sftp_port = urlparts.port;
7463 if(urlparts.port is None):
7464 sftp_port = 22;
7465 else:
7466 sftp_port = urlparts.port;
7467 if(urlparts.username is not None):
7468 sftp_username = urlparts.username;
7469 else:
7470 sftp_username = "anonymous";
7471 if(urlparts.password is not None):
7472 sftp_password = urlparts.password;
7473 elif(urlparts.password is None and urlparts.username=="anonymous"):
7474 sftp_password = "anonymous";
7475 else:
7476 sftp_password = "";
7477 if(urlparts.scheme=="ftp"):
7478 return download_file_from_ftp_file(url);
7479 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7480 return download_file_from_http_file(url);
7481 if(urlparts.scheme!="sftp"):
7482 return False;
7483 ssh = paramiko.SSHClient();
7484 ssh.load_system_host_keys();
7485 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7486 try:
7487 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7488 except paramiko.ssh_exception.SSHException:
7489 return False;
7490 except socket.gaierror:
7491 log.info("Error With URL "+url);
7492 return False;
7493 except socket.timeout:
7494 log.info("Error With URL "+url);
7495 return False;
7496 sftp = ssh.open_sftp();
7497 sftpfile = BytesIO();
7498 sftp.getfo(urlparts.path, sftpfile);
7499 sftp.close();
7500 ssh.close();
7501 sftpfile.seek(0, 0);
7502 return sftpfile;
7503 else:
7504 def download_file_from_sftp_file(url):
7505 return False;
7506
7507 if(haveparamiko):
7508 def download_file_from_sftp_string(url):
7509 sftpfile = download_file_from_sftp_file(url);
7510 return sftpfile.read();
7511 else:
7512 def download_file_from_ftp_string(url):
7513 return False;
7514
7515 if(haveparamiko):
7516 def upload_file_to_sftp_file(sftpfile, url):
7517 urlparts = urlparse(url);
7518 file_name = os.path.basename(urlparts.path);
7519 file_dir = os.path.dirname(urlparts.path);
7520 sftp_port = urlparts.port;
7521 if(urlparts.port is None):
7522 sftp_port = 22;
7523 else:
7524 sftp_port = urlparts.port;
7525 if(urlparts.username is not None):
7526 sftp_username = urlparts.username;
7527 else:
7528 sftp_username = "anonymous";
7529 if(urlparts.password is not None):
7530 sftp_password = urlparts.password;
7531 elif(urlparts.password is None and urlparts.username=="anonymous"):
7532 sftp_password = "anonymous";
7533 else:
7534 sftp_password = "";
7535 if(urlparts.scheme=="ftp"):
7536 return upload_file_to_ftp_file(url);
7537 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7538 return False;
7539 if(urlparts.scheme!="sftp"):
7540 return False;
7541 ssh = paramiko.SSHClient();
7542 ssh.load_system_host_keys();
7543 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7544 try:
7545 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7546 except paramiko.ssh_exception.SSHException:
7547 return False;
7548 except socket.gaierror:
7549 log.info("Error With URL "+url);
7550 return False;
7551 except socket.timeout:
7552 log.info("Error With URL "+url);
7553 return False;
7554 sftp = ssh.open_sftp();
7555 sftp.putfo(sftpfile, urlparts.path);
7556 sftp.close();
7557 ssh.close();
7558 sftpfile.seek(0, 0);
7559 return sftpfile;
7560 else:
7561 def upload_file_to_sftp_file(sftpfile, url):
7562 return False;
7563
7564 if(haveparamiko):
7565 def upload_file_to_sftp_string(sftpstring, url):
7566 sftpfileo = BytesIO(sftpstring);
7567 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7568 sftpfileo.close();
7569 return sftpfile;
7570 else:
7571 def upload_file_to_sftp_string(url):
7572 return False;
7573
7574 if(havepysftp):
7575 def download_file_from_pysftp_file(url):
7576 urlparts = urlparse(url);
7577 file_name = os.path.basename(urlparts.path);
7578 file_dir = os.path.dirname(urlparts.path);
7579 sftp_port = urlparts.port;
7580 if(urlparts.port is None):
7581 sftp_port = 22;
7582 else:
7583 sftp_port = urlparts.port;
7584 if(urlparts.username is not None):
7585 sftp_username = urlparts.username;
7586 else:
7587 sftp_username = "anonymous";
7588 if(urlparts.password is not None):
7589 sftp_password = urlparts.password;
7590 elif(urlparts.password is None and urlparts.username=="anonymous"):
7591 sftp_password = "anonymous";
7592 else:
7593 sftp_password = "";
7594 if(urlparts.scheme=="ftp"):
7595 return download_file_from_ftp_file(url);
7596 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7597 return download_file_from_http_file(url);
7598 if(urlparts.scheme!="sftp"):
7599 return False;
7600 try:
7601 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7602 except paramiko.ssh_exception.SSHException:
7603 return False;
7604 except socket.gaierror:
7605 log.info("Error With URL "+url);
7606 return False;
7607 except socket.timeout:
7608 log.info("Error With URL "+url);
7609 return False;
7610 sftp = ssh.open_sftp();
7611 sftpfile = BytesIO();
7612 sftp.getfo(urlparts.path, sftpfile);
7613 sftp.close();
7614 ssh.close();
7615 sftpfile.seek(0, 0);
7616 return sftpfile;
7617 else:
7618 def download_file_from_pysftp_file(url):
7619 return False;
7620
7621 if(havepysftp):
7622 def download_file_from_pysftp_string(url):
7623 sftpfile = download_file_from_pysftp_file(url);
7624 return sftpfile.read();
7625 else:
7626 def download_file_from_ftp_string(url):
7627 return False;
7628
7629 if(havepysftp):
7630 def upload_file_to_pysftp_file(sftpfile, url):
7631 urlparts = urlparse(url);
7632 file_name = os.path.basename(urlparts.path);
7633 file_dir = os.path.dirname(urlparts.path);
7634 sftp_port = urlparts.port;
7635 if(urlparts.port is None):
7636 sftp_port = 22;
7637 else:
7638 sftp_port = urlparts.port;
7639 if(urlparts.username is not None):
7640 sftp_username = urlparts.username;
7641 else:
7642 sftp_username = "anonymous";
7643 if(urlparts.password is not None):
7644 sftp_password = urlparts.password;
7645 elif(urlparts.password is None and urlparts.username=="anonymous"):
7646 sftp_password = "anonymous";
7647 else:
7648 sftp_password = "";
7649 if(urlparts.scheme=="ftp"):
7650 return upload_file_to_ftp_file(url);
7651 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7652 return False;
7653 if(urlparts.scheme!="sftp"):
7654 return False;
7655 try:
7656 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7657 except paramiko.ssh_exception.SSHException:
7658 return False;
7659 except socket.gaierror:
7660 log.info("Error With URL "+url);
7661 return False;
7662 except socket.timeout:
7663 log.info("Error With URL "+url);
7664 return False;
7665 sftp = ssh.open_sftp();
7666 sftp.putfo(sftpfile, urlparts.path);
7667 sftp.close();
7668 ssh.close();
7669 sftpfile.seek(0, 0);
7670 return sftpfile;
7671 else:
7672 def upload_file_to_pysftp_file(sftpfile, url):
7673 return False;
7674
7675 if(havepysftp):
7676 def upload_file_to_pysftp_string(sftpstring, url):
7677 sftpfileo = BytesIO(sftpstring);
7678 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7679 sftpfileo.close();
7680 return sftpfile;
7681 else:
7682 def upload_file_to_pysftp_string(url):
7683 return False;
7684
7685 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7686 urlparts = urlparse(url);
7687 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7688 return download_file_from_http_file(url, headers);
7689 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7690 return download_file_from_ftp_file(url);
7691 elif(urlparts.scheme=="sftp"):
7692 if(__use_pysftp__ and havepysftp):
7693 return download_file_from_pysftp_file(url);
7694 else:
7695 return download_file_from_sftp_file(url);
7696 else:
7697 return False;
7698 return False;
7699
7700 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7701 formatspecs = FormatSpecsListToDict(formatspecs);
7702 fp = download_file_from_internet_file(url);
7703 fp = UncompressArchiveFile(fp, formatspecs);
7704 fp.seek(0, 0);
7705 if(not fp):
7706 return False;
7707 return fp;
7708
7709 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7710 urlparts = urlparse(url);
7711 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7712 return download_file_from_http_string(url, headers);
7713 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7714 return download_file_from_ftp_string(url);
7715 elif(urlparts.scheme=="sftp"):
7716 if(__use_pysftp__ and havepysftp):
7717 return download_file_from_pysftp_string(url);
7718 else:
7719 return download_file_from_sftp_string(url);
7720 else:
7721 return False;
7722 return False;
7723
7724 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7725 formatspecs = FormatSpecsListToDict(formatspecs);
7726 fp = download_file_from_internet_string(url);
7727 fp = UncompressArchiveFile(fp, formatspecs);
7728 fp.seek(0, 0);
7729 if(not fp):
7730 return False;
7731 return fp;
7732
7733 def upload_file_to_internet_file(ifp, url):
7734 urlparts = urlparse(url);
7735 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7736 return False;
7737 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7738 return upload_file_to_ftp_file(ifp, url);
7739 elif(urlparts.scheme=="sftp"):
7740 if(__use_pysftp__ and havepysftp):
7741 return upload_file_to_pysftp_file(ifp, url);
7742 else:
7743 return upload_file_to_sftp_file(ifp, url);
7744 else:
7745 return False;
7746 return False;
7747
7748 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7749 formatspecs = FormatSpecsListToDict(formatspecs);
7750 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7751 if(not catfileout):
7752 return False;
7753 catfp.seek(0, 0);
7754 upload_file_to_internet_file(catfp, outfile);
7755 return True;
7756
7757 def upload_file_to_internet_string(ifp, url):
7758 urlparts = urlparse(url);
7759 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7760 return False;
7761 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7762 return upload_file_to_ftp_string(ifp, url);
7763 elif(urlparts.scheme=="sftp"):
7764 if(__use_pysftp__ and havepysftp):
7765 return upload_file_to_pysftp_string(ifp, url);
7766 else:
7767 return upload_file_to_sftp_string(ifp, url);
7768 else:
7769 return False;
7770 return False;
7771
7772 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7773 formatspecs = FormatSpecsListToDict(formatspecs);
7774 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7775 if(not catfileout):
7776 return False;
7777 catfp.seek(0, 0);
7778 upload_file_to_internet_file(catfp, outfile);
7779 return True;
7780
7781 try:
7782 if(hasattr(shutil, "register_archive_format")):
7783 # Register the packing format
7784 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7785 except shutil.RegistryError:
7786 pass;
7787
7788 try:
7789 if(hasattr(shutil, "register_unpack_format")):
7790 # Register the unpacking format
7791 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7792 except shutil.RegistryError:
7793 pass;
A tar like file format name catfile after unix cat command (concatenate files).