?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob9598e81b6b3e5192d076e320823dac399f95d849
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 6/2/2024 Ver. 0.12.4 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
38
39 # URL Parsing
40 try:
41 from urllib.parse import urlparse, urlunparse;
42 except ImportError:
43 from urlparse import urlparse, urlunparse;
44
45 # Windows-specific setup
46 if os.name == 'nt':
47 if sys.version_info[0] == 2:
48 import codecs;
49 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
50 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
51 else:
52 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
53 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
54
55 hashlib_guaranteed = False;
56 # Environment setup
57 os.environ["PYTHONIOENCODING"] = "UTF-8";
58 os.environ["LC_CTYPE"] = "UTF-8";
59
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys.version_info[0] == 2:
62 try:
63 reload(sys);
64 sys.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
66 pass;
67
68 # CRC32 import
69 try:
70 from zlib import crc32;
71 except ImportError:
72 from binascii import crc32;
73
74 # Define FileNotFoundError for Python 2
75 try:
76 FileNotFoundError;
77 except NameError:
78 FileNotFoundError = IOError;
79
80 # RAR file support
81 rarfile_support = False;
82 try:
83 import rarfile;
84 rarfile_support = True;
85 except ImportError:
86 pass;
87
88 # 7z file support
89 py7zr_support = False;
90 try:
91 import py7zr;
92 py7zr_support = True;
93 except ImportError:
94 pass;
95
96 # TAR file checking
97 try:
98 from xtarfile import is_tarfile;
99 except ImportError:
100 try:
101 from safetar import is_tarfile;
102 except ImportError:
103 from tarfile import is_tarfile;
104
105 # TAR file module
106 try:
107 import xtarfile as tarfile;
108 except ImportError:
109 try:
110 import safetar as tarfile;
111 except ImportError:
112 import tarfile;
113
114 # Paramiko support
115 haveparamiko = False;
116 try:
117 import paramiko;
118 haveparamiko = True;
119 except ImportError:
120 pass;
121
122 # PySFTP support
123 havepysftp = False;
124 try:
125 import pysftp;
126 havepysftp = True;
127 except ImportError:
128 pass;
129
130 # Requests support
131 haverequests = False;
132 try:
133 import requests;
134 haverequests = True;
135 import urllib3;
136 logging.getLogger("urllib3").setLevel(logging.WARNING);
137 except ImportError:
138 pass;
139
140 # HTTP and URL parsing
141 try:
142 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
143 from urllib.parse import urlparse;
144 except ImportError:
145 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
146 from urlparse import urlparse;
147
148 # StringIO and BytesIO
149 try:
150 from io import StringIO, BytesIO;
151 except ImportError:
152 try:
153 from cStringIO import StringIO;
154 from cStringIO import StringIO as BytesIO;
155 except ImportError:
156 from StringIO import StringIO;
157 from StringIO import StringIO as BytesIO;
158
159 __use_pysftp__ = False;
160 if(not havepysftp):
161 __use_pysftp__ = False;
162 __file_format_name__ = "CatFile";
163 __program_name__ = "Py"+__file_format_name__;
164 __file_format_lower__ = __file_format_name__.lower();
165 __file_format_magic__ = __file_format_name__;
166 __file_format_len__ = len(__file_format_magic__);
167 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__ = "\x00";
169 __file_format_ver__ = "001";
170 __use_new_style__ = True;
171 __use_advanced_list__ = True;
172 __use_alt_inode__ = False;
173 __file_format_extension__ = ".cat";
174 '''
175 __file_format_name__ = "FastArchive";
176 __program_name__ = "Py" + __file_format_name__;
177 __file_format_lower__ = __file_format_name__.lower();
178 __file_format_magic__ = "FstArch";
179 __file_format_len__ = len(__file_format_magic__);
180 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
181 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
182 __file_format_ver__ = "001";
183 __use_new_style__ = True;
184 __use_advanced_list__ = False;
185 __use_alt_inode__ = False;
186 __file_format_extension__ = ".fast";
187 '''
188 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
189 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
190 __project__ = __program_name__;
191 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
192 __version_info__ = (0, 12, 4, "RC 1", 1);
193 __version_date_info__ = (2024, 6, 2, "RC 1", 1);
194 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
195 __revision__ = __version_info__[3];
196 __revision_id__ = "$Id$";
197 if(__version_info__[4] is not None):
198 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
199 if(__version_info__[4] is None):
200 __version_date_plusrc__ = __version_date__;
201 if(__version_info__[3] is not None):
202 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
203 if(__version_info__[3] is None):
204 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
205
206 PyBitness = platform.architecture();
207 if(PyBitness=="32bit" or PyBitness=="32"):
208 PyBitness = "32";
209 elif(PyBitness=="64bit" or PyBitness=="64"):
210 PyBitness = "64";
211 else:
212 PyBitness = "32";
213
214 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
215 if(platform.python_implementation()!=""):
216 py_implementation = platform.python_implementation();
217 if(platform.python_implementation()==""):
218 py_implementation = "Python";
219 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
220 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
221 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
222 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
223 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
224 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
225 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
226
227 def CompressionSupport():
228 compression_list = [];
229 try:
230 import gzip;
231 compression_list.append("gz");
232 compression_list.append("gzip");
233 except ImportError:
234 '''return False;'''
235 try:
236 import bz2;
237 compression_list.append("bz2");
238 compression_list.append("bzip2");
239 except ImportError:
240 '''return False;'''
241 try:
242 import lz4;
243 compression_list.append("lz4");
244 except ImportError:
245 '''return False;'''
246 try:
247 import lzo;
248 compression_list.append("lzo");
249 compression_list.append("lzop");
250 except ImportError:
251 '''return False;'''
252 try:
253 import zstandard;
254 compression_list.append("zstd");
255 compression_list.append("zstandard");
256 except ImportError:
257 '''return False;'''
258 try:
259 import lzma;
260 compression_list.append("lzma");
261 compression_list.append("xz");
262 except ImportError:
263 '''return False;'''
264 try:
265 from backports import lzma;
266 compression_list.append("lzma");
267 compression_list.append("xz");
268 except ImportError:
269 '''return False;'''
270 return compression_list;
271
272 compressionsupport = CompressionSupport();
273 compressionlist = ['auto'];
274 compressionlistalt = [];
275 outextlist = [];
276 outextlistwd = [];
277 if('gzip' in compressionsupport):
278 compressionlist.append('gzip');
279 compressionlistalt.append('gzip');
280 outextlist.append('gz');
281 outextlistwd.append('.gz');
282 if('bzip2' in compressionsupport):
283 compressionlist.append('bzip2');
284 compressionlistalt.append('bzip2');
285 outextlist.append('bz2');
286 outextlistwd.append('.bz2');
287 if('zstd' in compressionsupport):
288 compressionlist.append('zstd');
289 compressionlistalt.append('zstd');
290 outextlist.append('zst');
291 outextlistwd.append('.zst');
292 if('lz4' in compressionsupport):
293 compressionlist.append('lz4');
294 compressionlistalt.append('lz4');
295 outextlist.append('lz4');
296 outextlistwd.append('.lz4');
297 if('lzo' in compressionsupport):
298 compressionlist.append('lzo');
299 compressionlistalt.append('lzo');
300 outextlist.append('lzo');
301 outextlistwd.append('.lzo');
302 if('lzop' in compressionsupport):
303 compressionlist.append('lzop');
304 compressionlistalt.append('lzop');
305 outextlist.append('lzop');
306 outextlistwd.append('.lzop');
307 if('lzma' in compressionsupport):
308 compressionlist.append('lzma');
309 compressionlistalt.append('lzma');
310 outextlist.append('lzma');
311 outextlistwd.append('.lzma');
312 if('xz' in compressionsupport):
313 compressionlist.append('xz');
314 compressionlistalt.append('xz');
315 outextlist.append('xz');
316 outextlistwd.append('.xz');
317
318 tarfile_mimetype = "application/tar";
319 tarfile_tar_mimetype = tarfile_mimetype;
320 zipfile_mimetype = "application/zip";
321 zipfile_zip_mimetype = zipfile_mimetype;
322 rarfile_mimetype = "application/rar";
323 rarfile_rar_mimetype = rarfile_mimetype;
324 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
325 archivefile_cat_mimetype = archivefile_mimetype;
326 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
327 archivefile_gz_mimetype = archivefile_gzip_mimetype;
328 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
329 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
330 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
331 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
332 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
333 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
334 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
335 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
336 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
337 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
338
339 if __name__ == "__main__":
340 import subprocess;
341 curscrpath = os.path.dirname(sys.argv[0]);
342 if(curscrpath==""):
343 curscrpath = ".";
344 if(os.sep=="\\"):
345 curscrpath = curscrpath.replace(os.sep, "/");
346 curscrpath = curscrpath + "/";
347 scrfile = curscrpath + "catfile.py";
348 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
349 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
350 scrcmd.wait();
351
352 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
353 if(not dbgenable):
354 return True;
355 log_functions = {
356 "print": print,
357 "log": logging.info,
358 "warning": logging.warning,
359 "error": logging.error,
360 "critical": logging.critical,
361 "exception": logging.exception,
362 "logalt": lambda x: logging.log(dgblevel, x),
363 "debug": logging.debug
364 };
365 log_function = log_functions.get(outtype);
366 if(log_function):
367 log_function(dbgtxt);
368 return True;
369 return False;
370
371 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
372 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
373 return dbgtxt;
374
375 def RemoveWindowsPath(dpath):
376 if(dpath is None):
377 dpath = "";
378 if(os.sep!="/"):
379 dpath = dpath.replace(os.path.sep, "/");
380 dpath = dpath.rstrip("/");
381 if(dpath=="." or dpath==".."):
382 dpath = dpath + "/";
383 return dpath;
384
385 def NormalizeRelativePath(inpath):
386 inpath = RemoveWindowsPath(inpath);
387 if(os.path.isabs(inpath)):
388 outpath = inpath;
389 else:
390 if(inpath.startswith("./") or inpath.startswith("../")):
391 outpath = inpath;
392 else:
393 outpath = "./" + inpath;
394 return outpath;
395
396 def PrependPath(base_dir, child_path):
397 # Check if base_dir is None or empty, if so, return child_path as is
398 if not base_dir:
399 return child_path;
400 # Ensure base_dir ends with exactly one slash
401 if not base_dir.endswith('/'):
402 base_dir += '/';
403 # Check if child_path starts with ./ or ../ (indicating a relative path)
404 if child_path.startswith('./') or child_path.startswith('../'):
405 # For relative paths, we don't alter the child_path
406 return base_dir + child_path;
407 else:
408 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
409 return base_dir + child_path.lstrip('/');
410
411 def ListDir(dirpath, followlink=False, duplicates=False):
412 if isinstance(dirpath, (list, tuple, )):
413 dirpath = list(filter(None, dirpath));
414 elif isinstance(dirpath, str):
415 dirpath = list(filter(None, [dirpath]));
416 retlist = [];
417 fs_encoding = sys.getfilesystemencoding();
418 for mydirfile in dirpath:
419 if not os.path.exists(mydirfile):
420 return False;
421 mydirfile = NormalizeRelativePath(mydirfile);
422 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
423 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
424 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
425 for root, dirs, filenames in os.walk(mydirfile):
426 dpath = root;
427 dpath = RemoveWindowsPath(dpath);
428 if fs_encoding != 'utf-8':
429 dpath = dpath.encode(fs_encoding).decode('utf-8');
430 if dpath not in retlist and not duplicates:
431 retlist.append(dpath);
432 if duplicates:
433 retlist.append(dpath);
434 for file in filenames:
435 fpath = os.path.join(root, file);
436 fpath = RemoveWindowsPath(fpath);
437 if fs_encoding != 'utf-8':
438 fpath = fpath.encode(fs_encoding).decode('utf-8');
439 if fpath not in retlist and not duplicates:
440 retlist.append(fpath);
441 if duplicates:
442 retlist.append(fpath);
443 else:
444 path = RemoveWindowsPath(mydirfile);
445 if fs_encoding != 'utf-8':
446 path = path.encode(fs_encoding).decode('utf-8');
447 retlist.append(path);
448 return retlist;
449
450 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
451 if isinstance(dirpath, (list, tuple, )):
452 dirpath = list(filter(None, dirpath));
453 elif isinstance(dirpath, str):
454 dirpath = list(filter(None, [dirpath]));
455 retlist = [];
456 fs_encoding = sys.getfilesystemencoding();
457 for mydirfile in dirpath:
458 if not os.path.exists(mydirfile):
459 return False;
460 mydirfile = NormalizeRelativePath(mydirfile);
461 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
462 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
463 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
464 for root, dirs, filenames in os.walk(mydirfile):
465 # Sort dirs and filenames alphabetically in place
466 dirs.sort(key=lambda x: x.lower());
467 filenames.sort(key=lambda x: x.lower());
468 dpath = RemoveWindowsPath(root);
469 if fs_encoding != 'utf-8':
470 dpath = dpath.encode(fs_encoding).decode('utf-8');
471 if not duplicates and dpath not in retlist:
472 retlist.append(dpath);
473 elif duplicates:
474 retlist.append(dpath);
475 for file in filenames:
476 fpath = os.path.join(root, file);
477 fpath = RemoveWindowsPath(fpath);
478 if fs_encoding != 'utf-8':
479 fpath = fpath.encode(fs_encoding).decode('utf-8');
480 if not duplicates and fpath not in retlist:
481 retlist.append(fpath);
482 elif duplicates:
483 retlist.append(fpath);
484 else:
485 path = RemoveWindowsPath(mydirfile);
486 if fs_encoding != 'utf-8':
487 path = path.encode(fs_encoding).decode('utf-8');
488 retlist.append(path);
489 return retlist;
490
491 def create_alias_function(prefix, base_name, suffix, target_function):
492 # Define a new function that wraps the target function
493 def alias_function(*args, **kwargs):
494 return target_function(*args, **kwargs);
495 # Create the function name by combining the prefix, base name, and the suffix
496 function_name = "{}{}{}".format(prefix, base_name, suffix);
497 # Add the new function to the global namespace
498 globals()[function_name] = alias_function;
499
500 def create_alias_function_alt(prefix, base_name, suffix, target_function):
501 # Create the function name by combining the prefix, base name, and the suffix
502 # Use the format method for string formatting, compatible with Python 2 and 3
503 function_name = "{}{}{}".format(prefix, base_name, suffix)
504 # Add the new function (alias of the target_function) to the global namespace
505 # This line is compatible as-is with both Python 2 and 3
506 globals()[function_name] = target_function
507
508 def FormatSpecsListToDict(formatspecs=__file_format_list__):
509 if(isinstance(formatspecs, (list, tuple, ))):
510 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
511 elif(isinstance(formatspecs, (dict, ))):
512 return formatspecs;
513 else:
514 return __file_format_dict__;
515 return __file_format_dict__;
516
517 def TarFileCheck(infile):
518 try:
519 if is_tarfile(infile):
520 return True;
521 else:
522 return False;
523 except TypeError:
524 try:
525 # Check if the input is a file object
526 if hasattr(infile, 'read'):
527 # Save the current file position
528 current_position = infile.tell();
529 # Attempt to open the file object as a tar file
530 tar = tarfile.open(fileobj=infile);
531 tar.close();
532 # Restore the file position
533 infile.seek(current_position);
534 return True;
535 else:
536 # Assume it's a filename
537 tar = tarfile.open(name=infile);
538 tar.close();
539 return True;
540 except tarfile.TarError:
541 return False;
542
543 # initial_value can be 0xFFFF or 0x0000
544 def crc16_ansi(msg, initial_value=0xFFFF):
545 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
546 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
547 crc = initial_value; # Initial value
548 for b in msg:
549 crc ^= b << 8; # XOR byte into CRC top byte
550 for _ in range(8): # Process each bit
551 if crc & 0x8000: # If the top bit is set
552 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
553 else:
554 crc = crc << 1; # Just shift left
555 crc &= 0xFFFF; # Ensure CRC remains 16-bit
556 return crc;
557
558 # initial_value can be 0xFFFF or 0x0000
559 def crc16_ibm(msg, initial_value=0xFFFF):
560 return crc16_ansi(msg, initial_value);
561
562 # initial_value is 0xFFFF
563 def crc16(msg):
564 return crc16_ansi(msg, 0xFFFF);
565
566 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
567 def crc16_ccitt(msg, initial_value=0xFFFF):
568 # CRC-16-CCITT polynomial
569 poly = 0x1021; # Polynomial for CRC-16-CCITT
570 # Use the specified initial value
571 crc = initial_value;
572 for b in msg:
573 crc ^= b << 8; # XOR byte into CRC top byte
574 for _ in range(8): # Process each bit
575 if crc & 0x8000: # If the top bit is set
576 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
577 else:
578 crc = crc << 1; # Just shift left
579 crc &= 0xFFFF; # Ensure CRC remains 16-bit
580 return crc;
581
582 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
583 def crc64_ecma(msg, initial_value=0x0000000000000000):
584 # CRC-64-ECMA polynomial and initial value
585 poly = 0x42F0E1EBA9EA3693;
586 crc = initial_value; # Initial value for CRC-64-ECMA
587 for b in msg:
588 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
589 for _ in range(8): # Process each bit
590 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
591 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
592 else:
593 crc <<= 1; # Just shift left if the MSB is 0
594 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
595 return crc;
596
597 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
598 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
599 # CRC-64-ISO polynomial and initial value
600 poly = 0x000000000000001B;
601 crc = initial_value; # Common initial value for CRC-64-ISO
602 for b in msg:
603 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
604 for _ in range(8): # Process each bit
605 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
606 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
607 else:
608 crc <<= 1; # Just shift left if the MSB is 0
609 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
610 return crc;
611
612 def GetDataFromArray(data, path, default=None):
613 element = data;
614 try:
615 for key in path:
616 element = element[key];
617 return element;
618 except (KeyError, TypeError, IndexError):
619 return default;
620
621 def GetDataFromArrayAlt(structure, path, default=None):
622 element = structure;
623 for key in path:
624 if isinstance(element, dict) and key in element:
625 element = element[key];
626 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
627 element = element[key];
628 else:
629 return default;
630 return element;
631
632 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
633 formatspecs = FormatSpecsListToDict(formatspecs);
634 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
635 if encodedata:
636 fileheader = fileheader.encode('UTF-8');
637 checksum_methods = {
638 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
639 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
640 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
641 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
642 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
643 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
644 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
645 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
646 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
647 };
648 if checksumtype in checksum_methods:
649 return checksum_methods[checksumtype](fileheader);
650 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
651 checksumoutstr = hashlib.new(checksumtype);
652 checksumoutstr.update(fileheader);
653 return checksumoutstr.hexdigest().lower();
654 return format(0, 'x').lower();
655
656 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
657 formatspecs = FormatSpecsListToDict(formatspecs);
658 if encodedata:
659 instr = instr.encode('UTF-8');
660 checksum_methods = {
661 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
662 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
663 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
664 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
665 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
666 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
667 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
668 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
669 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
670 };
671 if checksumtype in checksum_methods:
672 return checksum_methods[checksumtype](instr);
673 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
674 checksumoutstr = hashlib.new(checksumtype);
675 checksumoutstr.update(instr);
676 return checksumoutstr.hexdigest().lower();
677 return format(0, 'x').lower();
678
679 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
680 formatspecs = FormatSpecsListToDict(formatspecs);
681 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
682 return inchecksum.lower() == catfileheadercshex;
683
684 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
685 formatspecs = FormatSpecsListToDict(formatspecs);
686 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
687 return inchecksum.lower() == catinfilecshex;
688
689 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
690 curbyte = b"";
691 curfullbyte = b"";
692 nullbyte = delimiter.encode("UTF-8");
693 while(True):
694 curbyte = fp.read(1);
695 if(curbyte==nullbyte or not curbyte):
696 break;
697 curfullbyte = curfullbyte + curbyte;
698 return curfullbyte.decode('UTF-8');
699
700 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
701 return ReadTillNullByteOld(fp, delimiter);
702
703 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
704 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
705 buffer = bytearray();
706 total_read = 0;
707 delimiter_length = len(delimiter);
708 while True:
709 chunk = fp.read(chunk_size)
710 if not chunk:
711 # End of file reached without finding the delimiter
712 break;
713 buffer.extend(chunk);
714 total_read += len(chunk);
715 if delimiter in buffer:
716 # Delimiter found, calculate where to reset the file pointer
717 index = buffer.find(delimiter);
718 # Calculate how many extra bytes were read after the delimiter
719 extra_bytes_read = len(buffer) - (index + delimiter_length);
720 # Move the file pointer back to just after the delimiter
721 fp.seek(-extra_bytes_read, 1);
722 buffer = buffer[:index];
723 break;
724 if total_read >= max_read:
725 # Stop reading if max limit is reached to prevent excessive memory usage
726 raise MemoryError("Maximum read limit reached without finding the delimiter.");
727 # Check for incomplete UTF-8 sequences at the end of the buffer
728 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
729 # This suggests that the last byte might be the start of a multi-byte character
730 # Try to read one more byte to complete the character
731 extra_byte = fp.read(1);
732 if extra_byte:
733 buffer.extend(extra_byte);
734 else:
735 # No more data available
736 break;
737 try:
738 return buffer.decode('UTF-8', errors='replace');
739 except UnicodeDecodeError:
740 return buffer.decode('UTF-8', errors='replace');
741
742 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
743 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
744
745 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
746 curfullbyte = bytearray();
747 nullbyte = delimiter.encode("UTF-8");
748 total_read = 0; # Track the total number of bytes read
749 while True:
750 curbyte = fp.read(1);
751 if curbyte == nullbyte or not curbyte:
752 break;
753 curfullbyte.extend(curbyte);
754 total_read += 1;
755 if total_read >= max_read:
756 raise MemoryError("Maximum read limit reached without finding the delimiter.");
757 # Decode the full byte array to string once out of the loop
758 try:
759 return curfullbyte.decode('UTF-8');
760 except UnicodeDecodeError:
761 # Handle potential partial UTF-8 characters
762 for i in range(1, 4):
763 try:
764 return curfullbyte[:-i].decode('UTF-8');
765 except UnicodeDecodeError:
766 continue;
767 raise; # Re-raise if decoding fails even after trimming
768
769 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
770 return ReadTillNullByte(fp, delimiter, max_read);
771
772 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
773 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
774 buffer = bytearray();
775 total_read = 0;
776 delimiter_length = len(delimiter);
777 results = [];
778 while len(results) < num_delimiters:
779 chunk = fp.read(chunk_size);
780 if not chunk:
781 # End of file reached; decode whatever is collected if it's the last needed part
782 if len(buffer) > 0:
783 results.append(buffer.decode('UTF-8', errors='replace'));
784 break;
785 buffer.extend(chunk)
786 total_read += len(chunk);
787 # Check if we have found the delimiter
788 while delimiter in buffer:
789 index = buffer.find(delimiter);
790 # Decode the section before the delimiter
791 results.append(buffer[:index].decode('UTF-8', errors='replace'));
792 # Remove the processed part from the buffer
793 buffer = buffer[index + delimiter_length:];
794 if len(results) == num_delimiters:
795 # If reached the required number of delimiters, adjust the file pointer and stop
796 fp.seek(-len(buffer), 1);
797 return results;
798 if total_read >= max_read:
799 # Stop reading if max limit is reached to prevent excessive memory usage
800 raise MemoryError("Maximum read limit reached without finding the delimiter.");
801 # Check for incomplete UTF-8 sequences at the end of the buffer
802 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
803 # This suggests that the last byte might be the start of a multi-byte character
804 # Try to read one more byte to complete the character
805 extra_byte = fp.read(1);
806 if extra_byte:
807 buffer.extend(extra_byte);
808 else:
809 # No more data available
810 break;
811 # Process remaining buffer if less than the required number of delimiters were found
812 if len(buffer) > 0 and len(results) < num_delimiters:
813 results.append(buffer.decode('UTF-8', errors='replace'));
814 return results;
815
816 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
817 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
818
819 def SeekToEndOfFile(fp):
820 lasttell = 0;
821 while(True):
822 fp.seek(1, 1);
823 if(lasttell==fp.tell()):
824 break;
825 lasttell = fp.tell();
826 return True;
827
828 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
829 rocount = 0;
830 roend = int(rounds);
831 HeaderOut = [];
832 while(rocount<roend):
833 HeaderOut.append(ReadTillNullByte(fp, delimiter));
834 rocount = rocount + 1;
835 return HeaderOut;
836
837 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
838 headerpresize = ReadTillNullByte(fp, delimiter);
839 headersize = int(headerpresize, 16);
840 if(headersize<=0):
841 return [];
842 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
843 fp.seek(1, 1);
844 rocount = 0;
845 roend = int(len(headercontent));
846 HeaderOut = [headerpresize];
847 while(rocount<roend):
848 HeaderOut.append(headercontent[rocount]);
849 rocount = rocount + 1;
850 return HeaderOut;
851
852 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
853 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
854 headersize = int(preheaderdata[0], 16);
855 headernumfields = int(preheaderdata[1], 16);
856 if(headersize<=0 or headernumfields<=0):
857 return [];
858 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
859 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
860 HeaderOut = preheaderdata + headerdata;
861 return HeaderOut;
862
863 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
864 formatspecs = FormatSpecsListToDict(formatspecs);
865 delimiter = formatspecs['format_delimiter'];
866 fheaderstart = fp.tell();
867 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
868 if(len(HeaderOut)==0):
869 return False;
870 if(re.findall(r"^[.|/]", HeaderOut[3])):
871 fname = HeaderOut[3];
872 else:
873 fname = "./"+HeaderOut[3];
874 fcs = HeaderOut[-2].lower();
875 fccs = HeaderOut[-1].lower();
876 fsize = int(HeaderOut[5], 16);
877 fcompression = HeaderOut[12];
878 fcsize = int(HeaderOut[13], 16);
879 fseeknextfile = HeaderOut[25];
880 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
881 if(fcs!=newfcs and not skipchecksum):
882 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
883 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
884 return False;
885 fhend = fp.tell() - 1;
886 fcontentstart = fp.tell();
887 fcontents = BytesIO();
888 if(fsize>0 and not listonly):
889 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
890 fcontents.write(fp.read(fsize));
891 else:
892 fcontents.write(fp.read(fcsize));
893 elif(fsize>0 and listonly):
894 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
895 fp.seek(fsize, 1);
896 else:
897 fp.seek(fcsize, 1);
898 fcontents.seek(0, 0);
899 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
900 if(fccs!=newfccs and not skipchecksum and not listonly):
901 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
902 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
903 return False;
904 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
905 pass;
906 else:
907 fcontents.seek(0, 0);
908 if(uncompress):
909 fcontents = UncompressArchiveFile(fcontents, formatspecs);
910 fcontentend = fp.tell();
911 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
912 fseeknextasnum = int(fseeknextfile.replace("+", ""));
913 if(abs(fseeknextasnum)==0):
914 pass;
915 fp.seek(fseeknextasnum, 1);
916 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
917 fseeknextasnum = int(fseeknextfile);
918 if(abs(fseeknextasnum)==0):
919 pass;
920 fp.seek(fseeknextasnum, 1);
921 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
922 fseeknextasnum = int(fseeknextfile);
923 if(abs(fseeknextasnum)==0):
924 pass;
925 fp.seek(fseeknextasnum, 0);
926 else:
927 return False;
928 HeaderOut.append(fcontents);
929 return HeaderOut;
930
931 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
932 formatspecs = FormatSpecsListToDict(formatspecs);
933 delimiter = formatspecs['format_delimiter'];
934 fheaderstart = fp.tell();
935 if(formatspecs['new_style']):
936 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
937 else:
938 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
939 if(len(HeaderOut)==0):
940 return False;
941 fheadsize = int(HeaderOut[0], 16);
942 fnumfields = int(HeaderOut[1], 16);
943 ftype = int(HeaderOut[2], 16);
944 if(re.findall(r"^[.|/]", HeaderOut[3])):
945 fname = HeaderOut[3];
946 else:
947 fname = "./"+HeaderOut[3];
948 fbasedir = os.path.dirname(fname);
949 flinkname = HeaderOut[4];
950 fsize = int(HeaderOut[5], 16);
951 fatime = int(HeaderOut[6], 16);
952 fmtime = int(HeaderOut[7], 16);
953 fctime = int(HeaderOut[8], 16);
954 fbtime = int(HeaderOut[9], 16);
955 fmode = int(HeaderOut[10], 16);
956 fchmode = stat.S_IMODE(fmode);
957 ftypemod = stat.S_IFMT(fmode);
958 fwinattributes = int(HeaderOut[11], 16);
959 fcompression = HeaderOut[12];
960 fcsize = int(HeaderOut[13], 16);
961 fuid = int(HeaderOut[14], 16);
962 funame = HeaderOut[15];
963 fgid = int(HeaderOut[16], 16);
964 fgname = HeaderOut[17];
965 fid = int(HeaderOut[18], 16);
966 finode = int(HeaderOut[19], 16);
967 flinkcount = int(HeaderOut[20], 16);
968 fdev_minor = int(HeaderOut[21], 16);
969 fdev_major = int(HeaderOut[22], 16);
970 frdev_minor = int(HeaderOut[23], 16);
971 frdev_major = int(HeaderOut[24], 16);
972 fseeknextfile = HeaderOut[25];
973 fextrasize = int(HeaderOut[26], 16);
974 fextrafields = int(HeaderOut[27], 16);
975 extrafieldslist = [];
976 extrastart = 28
977 extraend = extrastart + fextrafields;
978 extrafieldslist = [];
979 if(extrastart<extraend):
980 extrafieldslist.append(HeaderOut[extrastart]);
981 extrastart = extrastart + 1;
982 fcs = HeaderOut[-2].lower();
983 fccs = HeaderOut[-1].lower();
984 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
985 if(fcs!=newfcs and not skipchecksum):
986 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
987 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
988 return False;
989 fhend = fp.tell() - 1;
990 fcontentstart = fp.tell();
991 fcontents = BytesIO();
992 pyhascontents = False;
993 if(fsize>0 and not listonly):
994 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
995 fcontents.write(fp.read(fsize));
996 else:
997 fcontents.write(fp.read(fcsize));
998 pyhascontents = True;
999 elif(fsize>0 and listonly):
1000 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1001 fp.seek(fsize, 1);
1002 else:
1003 fp.seek(fcsize, 1);
1004 pyhascontents = False;
1005 fcontents.seek(0, 0);
1006 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1007 if(fccs!=newfccs and not skipchecksum and not listonly):
1008 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1009 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1010 return False;
1011 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1012 pass;
1013 else:
1014 fcontents.seek(0, 0);
1015 if(uncompress):
1016 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1017 fcontents.seek(0, 0);
1018 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1019 fcontentend = fp.tell() - 1;
1020 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1021 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1022 if(abs(fseeknextasnum)==0):
1023 pass;
1024 fp.seek(fseeknextasnum, 1);
1025 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1026 fseeknextasnum = int(fseeknextfile);
1027 if(abs(fseeknextasnum)==0):
1028 pass;
1029 fp.seek(fseeknextasnum, 1);
1030 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1031 fseeknextasnum = int(fseeknextfile);
1032 if(abs(fseeknextasnum)==0):
1033 pass;
1034 fp.seek(fseeknextasnum, 0);
1035 else:
1036 return False;
1037 fcontents.seek(0, 0);
1038 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
1039 return catlist;
1040
1041 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1042 formatspecs = FormatSpecsListToDict(formatspecs);
1043 delimiter = formatspecs['format_delimiter'];
1044 fheaderstart = fp.tell();
1045 if(formatspecs['new_style']):
1046 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1047 else:
1048 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1049 if(len(HeaderOut)==0):
1050 return False;
1051 fheadsize = int(HeaderOut[0], 16);
1052 fnumfields = int(HeaderOut[1], 16);
1053 ftype = int(HeaderOut[2], 16);
1054 if(re.findall(r"^[.|/]", HeaderOut[3])):
1055 fname = HeaderOut[3];
1056 else:
1057 fname = "./"+HeaderOut[3];
1058 fbasedir = os.path.dirname(fname);
1059 flinkname = HeaderOut[4];
1060 fsize = int(HeaderOut[5], 16);
1061 fatime = int(HeaderOut[6], 16);
1062 fmtime = int(HeaderOut[7], 16);
1063 fctime = int(HeaderOut[8], 16);
1064 fbtime = int(HeaderOut[9], 16);
1065 fmode = int(HeaderOut[10], 16);
1066 fchmode = stat.S_IMODE(fmode);
1067 ftypemod = stat.S_IFMT(fmode);
1068 fwinattributes = int(HeaderOut[11], 16);
1069 fcompression = HeaderOut[12];
1070 fcsize = int(HeaderOut[13], 16);
1071 fuid = int(HeaderOut[14], 16);
1072 funame = HeaderOut[15];
1073 fgid = int(HeaderOut[16], 16);
1074 fgname = HeaderOut[17];
1075 fid = int(HeaderOut[18], 16);
1076 finode = int(HeaderOut[19], 16);
1077 flinkcount = int(HeaderOut[20], 16);
1078 fdev_minor = int(HeaderOut[21], 16);
1079 fdev_major = int(HeaderOut[22], 16);
1080 frdev_minor = int(HeaderOut[23], 16);
1081 frdev_major = int(HeaderOut[24], 16);
1082 fseeknextfile = HeaderOut[25];
1083 fextrasize = int(HeaderOut[26], 16);
1084 fextrafields = int(HeaderOut[27], 16);
1085 extrafieldslist = [];
1086 extrastart = 28;
1087 extraend = extrastart + fextrafields;
1088 extrafieldslist = [];
1089 if(extrastart<extraend):
1090 extrafieldslist.append(HeaderOut[extrastart]);
1091 extrastart = extrastart + 1;
1092 fheaderchecksumtype = HeaderOut[extrastart].lower();
1093 fcontentchecksumtype = HeaderOut[extrastart + 1].lower();
1094 fcs = HeaderOut[-2].lower();
1095 fccs = HeaderOut[-1].lower();
1096 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1097 if(fcs!=newfcs and not skipchecksum):
1098 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1099 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1100 return False;
1101 fhend = fp.tell() - 1;
1102 fcontentstart = fp.tell();
1103 fcontents = BytesIO();
1104 pyhascontents = False;
1105 if(fsize>0 and not listonly):
1106 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1107 fcontents.write(fp.read(fsize));
1108 else:
1109 fcontents.write(fp.read(fcsize));
1110 pyhascontents = True;
1111 elif(fsize>0 and listonly):
1112 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1113 fp.seek(fsize, 1);
1114 else:
1115 fp.seek(fcsize, 1);
1116 pyhascontents = False;
1117 fcontents.seek(0, 0);
1118 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1119 if(fccs!=newfccs and not skipchecksum and not listonly):
1120 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1121 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1122 return False;
1123 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1124 pass;
1125 else:
1126 fcontents.seek(0, 0);
1127 if(uncompress):
1128 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1129 fcontents.seek(0, 0);
1130 fcontentend = fp.tell() - 1;
1131 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1132 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1133 if(abs(fseeknextasnum)==0):
1134 pass;
1135 fp.seek(fseeknextasnum, 1);
1136 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1137 fseeknextasnum = int(fseeknextfile);
1138 if(abs(fseeknextasnum)==0):
1139 pass;
1140 fp.seek(fseeknextasnum, 1);
1141 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1142 fseeknextasnum = int(fseeknextfile);
1143 if(abs(fseeknextasnum)==0):
1144 pass;
1145 fp.seek(fseeknextasnum, 0);
1146 else:
1147 return False;
1148 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1149 return catlist;
1150
1151 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1152 formatspecs = FormatSpecsListToDict(formatspecs);
1153 delimiter = formatspecs['format_delimiter'];
1154 curloc = fp.tell();
1155 if(curloc>0):
1156 fp.seek(0, 0);
1157 catheader = ReadFileHeaderData(fp, 4, delimiter);
1158 if(curloc>0):
1159 fp.seek(curloc, 0);
1160 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1161 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1162 if(not headercheck and not skipchecksum):
1163 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1164 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1165 return False;
1166 fnumfiles = int(catheader[1], 16);
1167 countnum = 0;
1168 flist = [];
1169 while(countnum < fnumfiles):
1170 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1171 if(len(HeaderOut)==0):
1172 break;
1173 flist.append(HeaderOut);
1174 countnum = countnum + 1;
1175 return flist;
1176
1177 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1178 formatspecs = FormatSpecsListToDict(formatspecs);
1179 delimiter = formatspecs['format_delimiter'];
1180 curloc = fp.tell();
1181 if(curloc>0):
1182 fp.seek(0, 0);
1183 catheader = ReadFileHeaderData(fp, 4, delimiter);
1184 if(curloc>0):
1185 fp.seek(curloc, 0);
1186 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1187 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1188 if(not headercheck and not skipchecksum):
1189 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1190 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1191 return False;
1192 catstring = catheader[0];
1193 catversion = re.findall(r"([\d]+)", catstring);
1194 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1195 fprenumfiles = catheader[1];
1196 fnumfiles = int(fprenumfiles, 16);
1197 fprechecksumtype = catheader[2];
1198 fprechecksum = catheader[3];
1199 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1200 if(seekstart<0 and seekstart>fnumfiles):
1201 seekstart = 0;
1202 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1203 seekend = fnumfiles;
1204 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1205 seekend = fnumfiles - abs(seekend);
1206 if(seekstart>0):
1207 il = 0;
1208 while(il < seekstart):
1209 prefhstart = fp.tell();
1210 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1211 if(len(preheaderdata)==0):
1212 break;
1213 prefsize = int(preheaderdata[5], 16);
1214 prefseeknextfile = preheaderdata[25];
1215 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1216 prefcs = preheaderdata[-2];
1217 if(prefcs!=prenewfcs and not skipchecksum):
1218 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1219 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1220 return False;
1221 valid_archive = False;
1222 invalid_archive = True;
1223 prefhend = fp.tell() - 1;
1224 prefcontentstart = fp.tell();
1225 prefcontents = BytesIO();
1226 pyhascontents = False;
1227 if(prefsize>0):
1228 prefcontents.write(fp.read(prefsize));
1229 prefcontents.seek(0, 0);
1230 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1231 prefccs = preheaderdata[-1];
1232 pyhascontents = True;
1233 if(prefccs!=prenewfccs and not skipchecksum):
1234 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1235 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1236 return False;
1237 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1238 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1239 if(abs(fseeknextasnum)==0):
1240 pass;
1241 fp.seek(fseeknextasnum, 1);
1242 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1243 fseeknextasnum = int(prefseeknextfile);
1244 if(abs(fseeknextasnum)==0):
1245 pass;
1246 fp.seek(fseeknextasnum, 1);
1247 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1248 fseeknextasnum = int(prefseeknextfile);
1249 if(abs(fseeknextasnum)==0):
1250 pass;
1251 fp.seek(fseeknextasnum, 0);
1252 else:
1253 return False;
1254 il = il + 1;
1255 realidnum = 0;
1256 countnum = seekstart;
1257 while(countnum < seekend):
1258 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1259 if(len(HeaderOut)==0):
1260 break;
1261 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1262 catlist['ffilelist'].append(HeaderOut);
1263 countnum = countnum + 1;
1264 realidnum = realidnum + 1;
1265 return catlist;
1266
1267 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1268 formatspecs = FormatSpecsListToDict(formatspecs);
1269 delimiter = formatspecs['format_delimiter'];
1270 curloc = fp.tell();
1271 if(curloc>0):
1272 fp.seek(0, 0);
1273 catheader = ReadFileHeaderData(fp, 4, delimiter);
1274 if(curloc>0):
1275 fp.seek(curloc, 0);
1276 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1277 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1278 if(not headercheck and not skipchecksum):
1279 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1280 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1281 return False;
1282 catstring = catheader[0];
1283 catversion = re.findall(r"([\d]+)", catstring);
1284 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1285 fprenumfiles = catheader[1];
1286 fnumfiles = int(fprenumfiles, 16);
1287 fprechecksumtype = catheader[2];
1288 fprechecksum = catheader[3];
1289 catlist = [];
1290 if(seekstart<0 and seekstart>fnumfiles):
1291 seekstart = 0;
1292 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1293 seekend = fnumfiles;
1294 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1295 seekend = fnumfiles - abs(seekend);
1296 if(seekstart>0):
1297 il = 0;
1298 while(il < seekstart):
1299 prefhstart = fp.tell();
1300 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1301 if(len(preheaderdata)==0):
1302 break;
1303 prefsize = int(preheaderdata[5], 16);
1304 prefcompression = preheaderdata[12];
1305 prefcsize = int(preheaderdata[13], 16);
1306 prefseeknextfile = HeaderOut[25];
1307 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1308 prefcs = preheaderdata[-2];
1309 if(prefcs!=prenewfcs and not skipchecksum):
1310 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1311 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1312 return False;
1313 valid_archive = False;
1314 invalid_archive = True;
1315 prefhend = fp.tell() - 1;
1316 prefcontentstart = fp.tell();
1317 prefcontents = "";
1318 pyhascontents = False;
1319 if(prefsize>0):
1320 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1321 prefcontents = catfp.read(prefsize);
1322 else:
1323 prefcontents = catfp.read(prefcsize);
1324 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1325 prefccs = preheaderdata[-1];
1326 pyhascontents = True;
1327 if(prefccs!=prenewfccs and not skipchecksum):
1328 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1329 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1330 return False;
1331 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1332 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1333 if(abs(fseeknextasnum)==0):
1334 pass;
1335 catfp.seek(fseeknextasnum, 1);
1336 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1337 fseeknextasnum = int(prefseeknextfile);
1338 if(abs(fseeknextasnum)==0):
1339 pass;
1340 catfp.seek(fseeknextasnum, 1);
1341 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1342 fseeknextasnum = int(prefseeknextfile);
1343 if(abs(fseeknextasnum)==0):
1344 pass;
1345 catfp.seek(fseeknextasnum, 0);
1346 else:
1347 return False;
1348 il = il + 1;
1349 realidnum = 0;
1350 countnum = seekstart;
1351 while(countnum < seekend):
1352 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1353 if(len(HeaderOut)==0):
1354 break;
1355 catlist.append(HeaderOut);
1356 countnum = countnum + 1;
1357 realidnum = realidnum + 1;
1358 return catlist;
1359
1360 def ReadInFileBySizeWithContentToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1361 formatspecs = FormatSpecsListToDict(formatspecs);
1362 delimiter = formatspecs['format_delimiter'];
1363 if(hasattr(infile, "read") or hasattr(infile, "write")):
1364 fp = infile;
1365 fp.seek(0, 0);
1366 fp = UncompressArchiveFile(fp, formatspecs);
1367 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1368 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1369 return False;
1370 if(not fp):
1371 return False;
1372 fp.seek(0, 0);
1373 elif(infile=="-"):
1374 fp = BytesIO();
1375 if(hasattr(sys.stdin, "buffer")):
1376 shutil.copyfileobj(sys.stdin.buffer, fp);
1377 else:
1378 shutil.copyfileobj(sys.stdin, fp);
1379 fp.seek(0, 0);
1380 fp = UncompressArchiveFile(fp, formatspecs);
1381 if(not fp):
1382 return False;
1383 fp.seek(0, 0);
1384 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1385 fp = download_file_from_internet_file(infile);
1386 fp = UncompressArchiveFile(fp, formatspecs);
1387 fp.seek(0, 0);
1388 if(not fp):
1389 return False;
1390 fp.seek(0, 0);
1391 else:
1392 infile = RemoveWindowsPath(infile);
1393 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1394 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1395 return False;
1396 compresscheck = CheckCompressionType(infile, formatspecs, True);
1397 if(not compresscheck):
1398 fextname = os.path.splitext(infile)[1];
1399 if(fextname==".gz"):
1400 compresscheck = "gzip";
1401 elif(fextname==".bz2"):
1402 compresscheck = "bzip2";
1403 elif(fextname==".zst"):
1404 compresscheck = "zstd";
1405 elif(fextname==".lz4" or fextname==".clz4"):
1406 compresscheck = "lz4";
1407 elif(fextname==".lzo" or fextname==".lzop"):
1408 compresscheck = "lzo";
1409 elif(fextname==".lzma" or fextname==".xz"):
1410 compresscheck = "lzma";
1411 else:
1412 return False;
1413 if(not compresscheck):
1414 return False;
1415 fp = UncompressFile(infile, formatspecs, "rb");
1416 return ReadFileDataBySizeWithContentToArray(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1417
1418 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1419 formatspecs = FormatSpecsListToDict(formatspecs);
1420 delimiter = formatspecs['format_delimiter'];
1421 if(hasattr(infile, "read") or hasattr(infile, "write")):
1422 fp = infile;
1423 fp.seek(0, 0);
1424 fp = UncompressArchiveFile(fp, formatspecs);
1425 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1426 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1427 return False;
1428 if(not fp):
1429 return False;
1430 fp.seek(0, 0);
1431 elif(infile=="-"):
1432 fp = BytesIO();
1433 if(hasattr(sys.stdin, "buffer")):
1434 shutil.copyfileobj(sys.stdin.buffer, fp);
1435 else:
1436 shutil.copyfileobj(sys.stdin, fp);
1437 fp.seek(0, 0);
1438 fp = UncompressArchiveFile(fp, formatspecs);
1439 if(not fp):
1440 return False;
1441 fp.seek(0, 0);
1442 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1443 fp = download_file_from_internet_file(infile);
1444 fp = UncompressArchiveFile(fp, formatspecs);
1445 fp.seek(0, 0);
1446 if(not fp):
1447 return False;
1448 fp.seek(0, 0);
1449 else:
1450 infile = RemoveWindowsPath(infile);
1451 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1452 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1453 return False;
1454 compresscheck = CheckCompressionType(infile, formatspecs, True);
1455 if(not compresscheck):
1456 fextname = os.path.splitext(infile)[1];
1457 if(fextname==".gz"):
1458 compresscheck = "gzip";
1459 elif(fextname==".bz2"):
1460 compresscheck = "bzip2";
1461 elif(fextname==".zst"):
1462 compresscheck = "zstd";
1463 elif(fextname==".lz4" or fextname==".clz4"):
1464 compresscheck = "lz4";
1465 elif(fextname==".lzo" or fextname==".lzop"):
1466 compresscheck = "lzo";
1467 elif(fextname==".lzma" or fextname==".xz"):
1468 compresscheck = "lzma";
1469 else:
1470 return False;
1471 if(not compresscheck):
1472 return False;
1473 fp = UncompressFile(infile, formatspecs, "rb");
1474 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1475
1476 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1477 outdata = str(indata) + delimiter;
1478 return outdata;
1479
1480 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1481 outdata = "";
1482 inum = 0;
1483 il = len(indata);
1484 while(inum < il):
1485 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1486 inum = inum + 1;
1487 return outdata;
1488
1489 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1490 formatspecs = FormatSpecsListToDict(formatspecs);
1491 delimiter = formatspecs['format_delimiter'];
1492 catver = formatspecs['format_ver'];
1493 fileheaderver = str(int(catver.replace(".", "")));
1494 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1495 fp.write(fileheader.encode('UTF-8'));
1496 fnumfiles = format(int(numfiles), 'x').lower();
1497 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1498 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1499 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1500 fp.write(fnumfilesa.encode('UTF-8'));
1501 try:
1502 fp.flush();
1503 os.fsync(fp.fileno());
1504 except io.UnsupportedOperation:
1505 pass;
1506 except AttributeError:
1507 pass;
1508 except OSError as e:
1509 pass;
1510 return fp;
1511
1512 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1513 formatspecs = FormatSpecsListToDict(formatspecs);
1514 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1515 return fp;
1516
1517 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1518 formatspecs = FormatSpecsListToDict(formatspecs);
1519 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1520 if(os.path.exists(outfile)):
1521 try:
1522 os.unlink(outfile);
1523 except OSError as e:
1524 pass;
1525 if(outfile=="-"):
1526 verbose = False;
1527 catfpfp = BytesIO();
1528 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1529 catfp = outfile;
1530 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1531 catfp = BytesIO();
1532 else:
1533 fbasename = os.path.splitext(outfile)[0];
1534 fextname = os.path.splitext(outfile)[1];
1535 if(not compresswholefile and fextname in outextlistwd):
1536 compresswholefile = True;
1537 catfp = CompressOpenFile(outfile, True, compressionlevel);
1538 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1539 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1540 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1541 try:
1542 catfp.flush();
1543 os.fsync(catfp.fileno());
1544 except io.UnsupportedOperation:
1545 pass;
1546 except AttributeError:
1547 pass;
1548 except OSError as e:
1549 pass;
1550 if(outfile=="-"):
1551 catfp.seek(0, 0);
1552 if(hasattr(sys.stdout, "buffer")):
1553 shutil.copyfileobj(catfp, sys.stdout.buffer);
1554 else:
1555 shutil.copyfileobj(catfp, sys.stdout);
1556 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1557 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1558 catfp.seek(0, 0);
1559 upload_file_to_internet_file(catfp, outfile);
1560 if(returnfp):
1561 catfp.seek(0, 0);
1562 return catfp;
1563 else:
1564 catfp.close();
1565 return True;
1566
1567 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1568 formatspecs = FormatSpecsListToDict(formatspecs);
1569 extrafields = format(len(extradata), 'x').lower();
1570 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1571 if(len(extradata)>0):
1572 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1573 extrasizelen = format(len(extrasizestr), 'x').lower();
1574 catoutlen = len(filevalues) + len(extradata) + 5;
1575 catoutlenhex = format(catoutlen, 'x').lower();
1576 catoutlist = filevalues;
1577 catoutlist.insert(0, catoutlenhex);
1578 catoutlist.append(extrasizelen);
1579 catoutlist.append(extrafields);
1580 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1581 if(len(extradata)>0):
1582 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1583 if(len(filecontent)==0):
1584 checksumlist = [checksumtype, "none"];
1585 else:
1586 checksumlist = [checksumtype, checksumtype];
1587 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
1588 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1589 if(len(filecontent)==0):
1590 catfilecontentcshex = GetFileChecksum(filecontent, "none", False, formatspecs);
1591 else:
1592 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1593 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1594 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1595 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1596 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1597 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1598 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1599 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1600 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1601 fp.write(catfileout);
1602 try:
1603 fp.flush();
1604 os.fsync(fp.fileno());
1605 except io.UnsupportedOperation:
1606 pass;
1607 except AttributeError:
1608 pass;
1609 except OSError as e:
1610 pass;
1611 return fp;
1612
1613 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1614 formatspecs = FormatSpecsListToDict(formatspecs);
1615 advancedlist = formatspecs['use_advanced_list'];
1616 altinode = formatspecs['use_alt_inode'];
1617 if(verbose):
1618 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1619 if(infiles=="-"):
1620 for line in sys.stdin:
1621 infilelist.append(line.strip());
1622 infilelist = list(filter(None, infilelist));
1623 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1624 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1625 return False;
1626 with UncompressFile(infiles, formatspecs, "r") as finfile:
1627 for line in finfile:
1628 infilelist.append(line.strip());
1629 infilelist = list(filter(None, infilelist));
1630 else:
1631 if(isinstance(infiles, (list, tuple, ))):
1632 infilelist = list(filter(None, infiles));
1633 elif(isinstance(infiles, (str, ))):
1634 infilelist = list(filter(None, [infiles]));
1635 if(advancedlist):
1636 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1637 else:
1638 GetDirList = ListDir(infilelist, followlink, False);
1639 if(not GetDirList):
1640 return False;
1641 curinode = 0;
1642 curfid = 0;
1643 inodelist = [];
1644 inodetofile = {};
1645 filetoinode = {};
1646 inodetocatinode = {};
1647 numfiles = int(len(GetDirList));
1648 fnumfiles = format(numfiles, 'x').lower();
1649 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1650 for curfname in GetDirList:
1651 if(re.findall(r"^[.|/]", curfname)):
1652 fname = curfname;
1653 else:
1654 fname = "./"+curfname;
1655 if(verbose):
1656 VerbosePrintOut(fname);
1657 if(not followlink or followlink is None):
1658 fstatinfo = os.lstat(fname);
1659 else:
1660 fstatinfo = os.stat(fname);
1661 fpremode = fstatinfo.st_mode;
1662 finode = fstatinfo.st_ino;
1663 flinkcount = fstatinfo.st_nlink;
1664 ftype = 0;
1665 if(stat.S_ISREG(fpremode)):
1666 ftype = 0;
1667 elif(stat.S_ISLNK(fpremode)):
1668 ftype = 2;
1669 elif(stat.S_ISCHR(fpremode)):
1670 ftype = 3;
1671 elif(stat.S_ISBLK(fpremode)):
1672 ftype = 4;
1673 elif(stat.S_ISDIR(fpremode)):
1674 ftype = 5;
1675 elif(stat.S_ISFIFO(fpremode)):
1676 ftype = 6;
1677 elif(stat.S_ISSOCK(fpremode)):
1678 ftype = 8;
1679 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1680 ftype = 9;
1681 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1682 ftype = 10;
1683 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1684 ftype = 11;
1685 else:
1686 ftype = 0;
1687 flinkname = "";
1688 fcurfid = format(int(curfid), 'x').lower();
1689 if(not followlink and finode!=0):
1690 if(ftype!=1):
1691 if(finode in inodelist):
1692 ftype = 1;
1693 flinkname = inodetofile[finode];
1694 if(altinode):
1695 fcurinode = format(int(finode), 'x').lower();
1696 else:
1697 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1698 if(finode not in inodelist):
1699 inodelist.append(finode);
1700 inodetofile.update({finode: fname});
1701 inodetocatinode.update({finode: curinode});
1702 if(altinode):
1703 fcurinode = format(int(finode), 'x').lower();
1704 else:
1705 fcurinode = format(int(curinode), 'x').lower();
1706 curinode = curinode + 1;
1707 else:
1708 fcurinode = format(int(curinode), 'x').lower();
1709 curinode = curinode + 1;
1710 curfid = curfid + 1;
1711 if(ftype==2):
1712 flinkname = os.readlink(fname);
1713 fdev = fstatinfo.st_dev;
1714 getfdev = GetDevMajorMinor(fdev);
1715 fdev_minor = getfdev[0];
1716 fdev_major = getfdev[1];
1717 frdev = fstatinfo.st_dev;
1718 if(hasattr(fstatinfo, "st_rdev")):
1719 frdev = fstatinfo.st_rdev;
1720 else:
1721 frdev = fstatinfo.st_dev;
1722 getfrdev = GetDevMajorMinor(frdev);
1723 frdev_minor = getfrdev[0];
1724 frdev_major = getfrdev[1];
1725 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1726 fsize = format(int("0"), 'x').lower();
1727 elif(ftype==0 or ftype==7):
1728 fsize = format(int(fstatinfo.st_size), 'x').lower();
1729 else:
1730 fsize = format(int(fstatinfo.st_size)).lower();
1731 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1732 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1733 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1734 if(hasattr(fstatinfo, "st_birthtime")):
1735 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1736 else:
1737 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1738 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1739 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1740 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1741 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1742 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1743 funame = "";
1744 try:
1745 import pwd;
1746 try:
1747 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1748 funame = userinfo.pw_name;
1749 except KeyError:
1750 funame = "";
1751 except ImportError:
1752 funame = "";
1753 fgname = "";
1754 try:
1755 import grp;
1756 try:
1757 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1758 fgname = groupinfo.gr_name;
1759 except KeyError:
1760 fgname = "";
1761 except ImportError:
1762 fgname = "";
1763 fdev_minor = format(int(fdev_minor), 'x').lower();
1764 fdev_major = format(int(fdev_major), 'x').lower();
1765 frdev_minor = format(int(frdev_minor), 'x').lower();
1766 frdev_major = format(int(frdev_major), 'x').lower();
1767 finode = format(int(finode), 'x').lower();
1768 flinkcount = format(int(flinkcount), 'x').lower();
1769 if(hasattr(fstatinfo, "st_file_attributes")):
1770 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1771 else:
1772 fwinattributes = format(int(0), 'x').lower();
1773 fcompression = "";
1774 fcsize = format(int(0), 'x').lower();
1775 fcontents = BytesIO();
1776 chunk_size = 1024;
1777 if(ftype==0 or ftype==7):
1778 with open(fname, "rb") as fpc:
1779 shutil.copyfileobj(fpc, fcontents);
1780 if(not compresswholefile):
1781 fcontents.seek(0, 2);
1782 ucfsize = fcontents.tell();
1783 fcontents.seek(0, 0);
1784 if(compression=="auto"):
1785 ilsize = len(compressionlistalt);
1786 ilmin = 0;
1787 ilcsize = [];
1788 while(ilmin < ilsize):
1789 cfcontents = BytesIO();
1790 shutil.copyfileobj(fcontents, cfcontents);
1791 fcontents.seek(0, 0);
1792 cfcontents.seek(0, 0);
1793 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1794 if(cfcontents):
1795 cfcontents.seek(0, 2);
1796 ilcsize.append(cfcontents.tell());
1797 cfcontents.close();
1798 else:
1799 try:
1800 ilcsize.append(sys.maxint);
1801 except AttributeError:
1802 ilcsize.append(sys.maxsize);
1803 ilmin = ilmin + 1;
1804 ilcmin = ilcsize.index(min(ilcsize));
1805 compression = compressionlistalt[ilcmin];
1806 fcontents.seek(0, 0);
1807 cfcontents = BytesIO();
1808 shutil.copyfileobj(fcontents, cfcontents);
1809 cfcontents.seek(0, 0);
1810 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1811 cfcontents.seek(0, 2);
1812 cfsize = cfcontents.tell();
1813 if(ucfsize > cfsize):
1814 fcsize = format(int(cfsize), 'x').lower();
1815 fcompression = compression;
1816 fcontents.close();
1817 fcontents = cfcontents;
1818 if(followlink and (ftype==1 or ftype==2)):
1819 flstatinfo = os.stat(flinkname);
1820 with open(flinkname, "rb") as fpc:
1821 shutil.copyfileobj(fpc, fcontents);
1822 if(not compresswholefile):
1823 fcontents.seek(0, 2);
1824 ucfsize = fcontents.tell();
1825 fcontents.seek(0, 0);
1826 if(compression=="auto"):
1827 ilsize = len(compressionlistalt);
1828 ilmin = 0;
1829 ilcsize = [];
1830 while(ilmin < ilsize):
1831 cfcontents = BytesIO();
1832 shutil.copyfileobj(fcontents, cfcontents);
1833 fcontents.seek(0, 0);
1834 cfcontents.seek(0, 0);
1835 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1836 if(cfcontents):
1837 cfcontents.seek(0, 2);
1838 ilcsize.append(cfcontents.tell());
1839 cfcontents.close();
1840 else:
1841 try:
1842 ilcsize.append(sys.maxint);
1843 except AttributeError:
1844 ilcsize.append(sys.maxsize);
1845 ilmin = ilmin + 1;
1846 ilcmin = ilcsize.index(min(ilcsize));
1847 compression = compressionlistalt[ilcmin];
1848 fcontents.seek(0, 0);
1849 cfcontents = BytesIO();
1850 shutil.copyfileobj(fcontents, cfcontents);
1851 cfcontents.seek(0, 0);
1852 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1853 cfcontents.seek(0, 2);
1854 cfsize = cfcontents.tell();
1855 if(ucfsize > cfsize):
1856 fcsize = format(int(cfsize), 'x').lower();
1857 fcompression = compression;
1858 fcontents.close();
1859 fcontents = cfcontents;
1860 if(fcompression=="none"):
1861 fcompression = "";
1862 fcontents.seek(0, 0);
1863 ftypehex = format(ftype, 'x').lower();
1864 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
1865 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1866 if(numfiles>0):
1867 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1868 fp.seek(0, 0);
1869 return fp;
1870
1871 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1872 formatspecs = FormatSpecsListToDict(formatspecs);
1873 if(verbose):
1874 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1875 GetDirList = inlist;
1876 if(not GetDirList):
1877 return False;
1878 curinode = 0;
1879 curfid = 0;
1880 inodelist = [];
1881 inodetofile = {};
1882 filetoinode = {};
1883 inodetocatinode = {};
1884 numfiles = int(len(GetDirList));
1885 fnumfiles = format(numfiles, 'x').lower();
1886 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1887 for curfname in GetDirList:
1888 ftype = format(curfname[0], 'x').lower();
1889 if(re.findall(r"^[.|/]", curfname[1])):
1890 fname = curfname[1];
1891 else:
1892 fname = "./"+curfname[1];
1893 fbasedir = os.path.dirname(fname);
1894 flinkname = curfname[2];
1895 fsize = format(curfname[3], 'x').lower();
1896 fatime = format(curfname[4], 'x').lower();
1897 fmtime = format(curfname[5], 'x').lower();
1898 fctime = format(curfname[6], 'x').lower();
1899 fbtime = format(curfname[7], 'x').lower();
1900 fmode = format(curfname[8], 'x').lower();
1901 fwinattributes = format(curfname[9], 'x').lower();
1902 fcompression = curfname[10];
1903 fcsize = format(curfname[11], 'x').lower();
1904 fuid = format(curfname[12], 'x').lower();
1905 funame = curfname[13];
1906 fgid = format(curfname[14], 'x').lower();
1907 fgname = curfname[15];
1908 fid = format(curfname[16], 'x').lower();
1909 finode = format(curfname[17], 'x').lower();
1910 flinkcount = format(curfname[18], 'x').lower();
1911 fdev_minor = format(curfname[19], 'x').lower();
1912 fdev_major = format(curfname[20], 'x').lower();
1913 frdev_minor = format(curfname[21], 'x').lower();
1914 frdev_major = format(curfname[22], 'x').lower();
1915 fseeknextfile = curfname[23];
1916 extradata = curfname[24];
1917 fheaderchecksumtype = curfname[25];
1918 fcontentchecksumtype = curfname[26];
1919 fcontents = curfname[27];
1920 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
1921 fcontents.seek(0, 0);
1922 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1923 if(numfiles>0):
1924 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1925 return fp;
1926
1927 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1928 formatspecs = FormatSpecsListToDict(formatspecs);
1929 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1930 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1931
1932 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1933 formatspecs = FormatSpecsListToDict(formatspecs);
1934 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1935 if(os.path.exists(outfile)):
1936 try:
1937 os.unlink(outfile);
1938 except OSError as e:
1939 pass;
1940 if(outfile=="-"):
1941 verbose = False;
1942 catfpfp = BytesIO();
1943 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1944 catfp = outfile;
1945 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1946 catfp = BytesIO();
1947 else:
1948 fbasename = os.path.splitext(outfile)[0];
1949 fextname = os.path.splitext(outfile)[1];
1950 if(not compresswholefile and fextname in outextlistwd):
1951 compresswholefile = True;
1952 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1953 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1954 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1955 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1956 try:
1957 catfp.flush();
1958 os.fsync(catfp.fileno());
1959 except io.UnsupportedOperation:
1960 pass;
1961 except AttributeError:
1962 pass;
1963 except OSError as e:
1964 pass;
1965 if(outfile=="-"):
1966 catfp.seek(0, 0);
1967 if(hasattr(sys.stdout, "buffer")):
1968 shutil.copyfileobj(catfp, sys.stdout.buffer);
1969 else:
1970 shutil.copyfileobj(catfp, sys.stdout);
1971 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1972 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1973 catfp.seek(0, 0);
1974 upload_file_to_internet_file(catfp, outfile);
1975 if(returnfp):
1976 catfp.seek(0, 0);
1977 return catfp;
1978 else:
1979 catfp.close();
1980 return True;
1981
1982 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1983 formatspecs = FormatSpecsListToDict(formatspecs);
1984 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1985 if(os.path.exists(outfile)):
1986 try:
1987 os.unlink(outfile);
1988 except OSError as e:
1989 pass;
1990 if(outfile=="-"):
1991 verbose = False;
1992 catfpfp = BytesIO();
1993 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1994 catfp = outfile;
1995 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1996 catfp = BytesIO();
1997 else:
1998 fbasename = os.path.splitext(outfile)[0];
1999 fextname = os.path.splitext(outfile)[1];
2000 if(not compresswholefile and fextname in outextlistwd):
2001 compresswholefile = True;
2002 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2003 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2004 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2005 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2006 try:
2007 catfp.flush();
2008 os.fsync(catfp.fileno());
2009 except io.UnsupportedOperation:
2010 pass;
2011 except AttributeError:
2012 pass;
2013 except OSError as e:
2014 pass;
2015 if(outfile=="-"):
2016 catfp.seek(0, 0);
2017 if(hasattr(sys.stdout, "buffer")):
2018 shutil.copyfileobj(catfp, sys.stdout.buffer);
2019 else:
2020 shutil.copyfileobj(catfp, sys.stdout);
2021 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2022 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2023 catfp.seek(0, 0);
2024 upload_file_to_internet_file(catfp, outfile);
2025 if(returnfp):
2026 catfp.seek(0, 0);
2027 return catfp;
2028 else:
2029 catfp.close();
2030 return True;
2031
2032 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2033 formatspecs = FormatSpecsListToDict(formatspecs);
2034 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2035 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2036
2037 def PrintPermissionString(fchmode, ftype):
2038 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2039 permissionstr = "";
2040 for fmodval in str(oct(fchmode))[-3:]:
2041 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2042 if(ftype==0 or ftype==7):
2043 permissionstr = "-" + permissionstr;
2044 if(ftype==1):
2045 permissionstr = "h" + permissionstr;
2046 if(ftype==2):
2047 permissionstr = "l" + permissionstr;
2048 if(ftype==3):
2049 permissionstr = "c" + permissionstr;
2050 if(ftype==4):
2051 permissionstr = "b" + permissionstr;
2052 if(ftype==5):
2053 permissionstr = "d" + permissionstr;
2054 if(ftype==6):
2055 permissionstr = "f" + permissionstr;
2056 if(ftype==8):
2057 permissionstr = "D" + permissionstr;
2058 if(ftype==9):
2059 permissionstr = "p" + permissionstr;
2060 if(ftype==10):
2061 permissionstr = "w" + permissionstr;
2062 try:
2063 permissionoutstr = stat.filemode(fchmode);
2064 except AttributeError:
2065 permissionoutstr = permissionstr;
2066 except KeyError:
2067 permissionoutstr = permissionstr;
2068 return permissionoutstr;
2069
2070 def PrintPermissionStringAlt(fchmode, ftype):
2071 permissions = {
2072 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2073 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2074 };
2075 # Translate file mode into permission string
2076 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2077 # Append file type indicator
2078 type_indicators = {
2079 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2080 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2081 };
2082 file_type = type_indicators.get(ftype, '-');
2083 permissionstr = file_type + permissionstr;
2084 try:
2085 permissionoutstr = stat.filemode(fchmode);
2086 except AttributeError:
2087 permissionoutstr = permissionstr;
2088 return permissionoutstr;
2089
2090 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2091 formatspecs = FormatSpecsListToDict(formatspecs);
2092 if(hasattr(infile, "read") or hasattr(infile, "write")):
2093 catfp = infile;
2094 else:
2095 try:
2096 catfp = open(infile, "rb");
2097 except FileNotFoundError:
2098 return False;
2099 catfp.seek(0, 0);
2100 prefp = catfp.read(2);
2101 filetype = False;
2102 if(prefp==binascii.unhexlify("1f8b")):
2103 filetype = "gzip";
2104 catfp.seek(0, 0);
2105 prefp = catfp.read(3);
2106 if(prefp==binascii.unhexlify("425a68")):
2107 filetype = "bzip2";
2108 if(prefp==binascii.unhexlify("5d0000")):
2109 filetype = "lzma";
2110 catfp.seek(0, 0);
2111 prefp = catfp.read(4);
2112 if(prefp==binascii.unhexlify("28b52ffd")):
2113 filetype = "zstd";
2114 if(prefp==binascii.unhexlify("04224d18")):
2115 filetype = "lz4";
2116 if(prefp==binascii.unhexlify("504B0304")):
2117 filetype = "zipfile";
2118 catfp.seek(0, 0);
2119 prefp = catfp.read(5);
2120 if(prefp==binascii.unhexlify("7573746172")):
2121 filetype = "tarfile";
2122 catfp.seek(0, 0);
2123 prefp = catfp.read(6);
2124 if(prefp==binascii.unhexlify("fd377a585a00")):
2125 filetype = "lzma";
2126 if(prefp==binascii.unhexlify("377abcaf271c")):
2127 filetype = "7zipfile";
2128 catfp.seek(0, 0);
2129 prefp = catfp.read(7);
2130 if(prefp==binascii.unhexlify("526172211a0700")):
2131 filetype = "rarfile";
2132 if(prefp==binascii.unhexlify("43617446696c65")):
2133 filetype = "catfile";
2134 catfp.seek(0, 0);
2135 prefp = catfp.read(8);
2136 if(prefp==binascii.unhexlify("526172211a070100")):
2137 filetype = "rarfile";
2138 catfp.seek(0, 0);
2139 prefp = catfp.read(formatspecs['format_len']);
2140 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2141 filetype = formatspecs['format_lower'];
2142 catfp.seek(0, 0);
2143 prefp = catfp.read(9);
2144 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2145 filetype = "lzo";
2146 catfp.seek(0, 0);
2147 prefp = catfp.read(10);
2148 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2149 filetype = "tarfile";
2150 catfp.seek(0, 0);
2151 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2152 if(TarFileCheck(catfp)):
2153 filetype = "tarfile";
2154 if(not filetype):
2155 if(TarFileCheck(catfp)):
2156 filetype = "tarfile";
2157 elif(zipfile.is_zipfile(catfp)):
2158 filetype = "zipfile";
2159 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2160 filetype = "rarile";
2161 else:
2162 filetype = False;
2163 catfp.seek(0, 0);
2164 if(closefp):
2165 catfp.close();
2166 return filetype;
2167
2168 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2169 formatspecs = FormatSpecsListToDict(formatspecs);
2170 try:
2171 instringsfile = BytesIO(instring);
2172 except TypeError:
2173 instringsfile = BytesIO(instring.encode("UTF-8"));
2174 return CheckCompressionType(instringsfile, formatspecs, closefp);
2175
2176 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2177 formatspecs = FormatSpecsListToDict(formatspecs);
2178 compresscheck = CheckCompressionType(fp, formatspecs, False);
2179 if(compresscheck=="gzip" or compresscheck=="gz"):
2180 return archivefile_gzip_mimetype;
2181 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2182 return archivefile_bzip2_mimetype;
2183 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2184 return archivefile_zstandard_mimetype;
2185 if(compresscheck=="lz4"):
2186 return archivefile_lz4_mimetype;
2187 if(compresscheck=="lzo" or compresscheck=="lzop"):
2188 return archivefile_lzop_mimetype;
2189 if(compresscheck=="lzma"):
2190 return archivefile_lzma_mimetype;
2191 if(compresscheck=="xz"):
2192 return archivefile_xz_mimetype;
2193 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2194 return archivefile_cat_mimetype;
2195 if(not compresscheck):
2196 return False;
2197 return False;
2198
2199 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2200 formatspecs = FormatSpecsListToDict(formatspecs);
2201 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2202 return False;
2203 compresscheck = CheckCompressionType(fp, formatspecs, False);
2204 if(compresscheck=="gzip"):
2205 try:
2206 import gzip;
2207 except ImportError:
2208 return False;
2209 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2210 if(compresscheck=="bzip2"):
2211 try:
2212 import bz2;
2213 except ImportError:
2214 return False;
2215 catfp = BytesIO();
2216 catfp.write(bz2.decompress(fp.read()));
2217 if(compresscheck=="zstd"):
2218 try:
2219 import zstandard;
2220 except ImportError:
2221 return False;
2222 catfp = BytesIO();
2223 catfp.write(zstandard.decompress(fp.read()));
2224 if(compresscheck=="lz4"):
2225 try:
2226 import lz4.frame;
2227 except ImportError:
2228 return False;
2229 catfp = BytesIO();
2230 catfp.write(lz4.frame.decompress(fp.read()));
2231 if(compresscheck=="lzo" or compresscheck=="lzop"):
2232 try:
2233 import lzo;
2234 except ImportError:
2235 return False;
2236 catfp = BytesIO();
2237 catfp.write(lzo.decompress(fp.read()));
2238 if(compresscheck=="lzma" or compresscheck=="xz"):
2239 try:
2240 import lzma;
2241 except ImportError:
2242 try:
2243 from backports import lzma
2244 except ImportError:
2245 return False;
2246 catfp = BytesIO();
2247 catfp.write(lzma.decompress(fp.read()));
2248 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2249 catfp = fp;
2250 if(not compresscheck):
2251 try:
2252 import lzma;
2253 except ImportError:
2254 try:
2255 from backports import lzma
2256 except ImportError:
2257 return False;
2258 catfp = BytesIO();
2259 with fp as fpcontent:
2260 try:
2261 catfp.write(lzma.decompress(fp.read()));
2262 except lzma.LZMAError:
2263 return False;
2264 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2265 fp.close();
2266 return catfp;
2267
2268 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2269
2270 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2271 formatspecs = FormatSpecsListToDict(formatspecs);
2272 compresscheck = CheckCompressionType(infile, formatspecs, False);
2273 if(sys.version_info[0]==2 and compresscheck):
2274 if(mode=="rt"):
2275 mode = "r";
2276 if(mode=="wt"):
2277 mode = "w";
2278 try:
2279 if(compresscheck=="gzip"):
2280 try:
2281 import gzip;
2282 except ImportError:
2283 return False;
2284 try:
2285 filefp = gzip.open(infile, mode, encoding="UTF-8");
2286 except (ValueError, TypeError) as e:
2287 filefp = gzip.open(infile, mode);
2288 if(compresscheck=="bzip2"):
2289 try:
2290 import bz2;
2291 except ImportError:
2292 return False;
2293 try:
2294 filefp = bz2.open(infile, mode, encoding="UTF-8");
2295 except (ValueError, TypeError) as e:
2296 filefp = bz2.open(infile, mode);
2297 if(compresscheck=="zstd"):
2298 try:
2299 import zstandard;
2300 except ImportError:
2301 return False;
2302 try:
2303 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2304 except (ValueError, TypeError) as e:
2305 filefp = zstandard.open(infile, mode);
2306 if(compresscheck=="lz4"):
2307 try:
2308 import lz4.frame;
2309 except ImportError:
2310 return False;
2311 try:
2312 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2313 except (ValueError, TypeError) as e:
2314 filefp = lz4.frame.open(infile, mode);
2315 if(compresscheck=="lzo"):
2316 try:
2317 import lzo;
2318 except ImportError:
2319 return False;
2320 try:
2321 filefp = lzo.open(infile, mode, encoding="UTF-8");
2322 except (ValueError, TypeError) as e:
2323 filefp = lzo.open(infile, mode);
2324 if(compresscheck=="lzma"):
2325 try:
2326 import lzma;
2327 except ImportError:
2328 try:
2329 from backports import lzma
2330 except ImportError:
2331 return False;
2332 try:
2333 filefp = lzma.open(infile, mode, encoding="UTF-8");
2334 except (ValueError, TypeError) as e:
2335 filefp = lzma.open(infile, mode);
2336 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2337 try:
2338 filefp = open(infile, mode, encoding="UTF-8");
2339 except (ValueError, TypeError) as e:
2340 filefp = open(infile, mode);
2341 if(not compresscheck):
2342 try:
2343 filefp = open(infile, mode, encoding="UTF-8");
2344 except (ValueError, TypeError) as e:
2345 filefp = open(infile, mode);
2346 except FileNotFoundError:
2347 return False;
2348 return filefp;
2349
2350 def UncompressString(infile):
2351 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2352 if(compresscheck=="gzip"):
2353 try:
2354 import gzip;
2355 except ImportError:
2356 return False;
2357 fileuz = gzip.decompress(infile);
2358 if(compresscheck=="bzip2"):
2359 try:
2360 import bz2;
2361 except ImportError:
2362 return False;
2363 fileuz = bz2.decompress(infile);
2364 if(compresscheck=="zstd"):
2365 try:
2366 import zstandard;
2367 except ImportError:
2368 return False;
2369 fileuz = zstandard.decompress(infile);
2370 if(compresscheck=="lz4"):
2371 try:
2372 import lz4.frame;
2373 except ImportError:
2374 return False;
2375 fileuz = lz4.frame.decompress(infile);
2376 if(compresscheck=="lzo"):
2377 try:
2378 import lzo;
2379 except ImportError:
2380 return False;
2381 fileuz = lzo.decompress(infile);
2382 if(compresscheck=="lzma"):
2383 try:
2384 import lzma;
2385 except ImportError:
2386 try:
2387 from backports import lzma
2388 except ImportError:
2389 return False;
2390 fileuz = lzma.decompress(infile);
2391 if(not compresscheck):
2392 fileuz = infile;
2393 if(hasattr(fileuz, 'decode')):
2394 fileuz = fileuz.decode("UTF-8");
2395 return fileuz;
2396
2397 def UncompressStringAlt(infile):
2398 filefp = StringIO();
2399 outstring = UncompressString(infile);
2400 filefp.write(outstring);
2401 filefp.seek(0, 0);
2402 return filefp;
2403
2404 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2405 formatspecs = FormatSpecsListToDict(formatspecs);
2406 compresscheck = CheckCompressionType(infile, formatspecs, False);
2407 if(not compresscheck):
2408 fextname = os.path.splitext(infile)[1];
2409 if(fextname==".gz"):
2410 compresscheck = "gzip";
2411 elif(fextname==".bz2"):
2412 compresscheck = "bzip2";
2413 elif(fextname==".zst"):
2414 compresscheck = "zstd";
2415 elif(fextname==".lz4"):
2416 compresscheck = "lz4";
2417 elif(fextname==".lzo" or fextname==".lzop"):
2418 compresscheck = "lzo";
2419 elif(fextname==".lzma" or fextname==".xz"):
2420 compresscheck = "lzma";
2421 else:
2422 return False;
2423 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2424 if(TarFileCheck(infile)):
2425 filetype = "tarfile";
2426 if(not compresscheck):
2427 if(TarFileCheck(infile)):
2428 return "tarfile";
2429 elif(zipfile.is_zipfile(infile)):
2430 return "zipfile";
2431 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2432 return "rarile";
2433 else:
2434 return False;
2435 return False;
2436 if(compresscheck=="catfile"):
2437 return "catfile";
2438 if(compresscheck==formatspecs['format_lower']):
2439 return formatspecs['format_lower'];
2440 if(compresscheck=="tarfile"):
2441 return "tarfile";
2442 if(compresscheck=="zipfile"):
2443 return "zipfile";
2444 if(rarfile_support and compresscheck=="rarfile"):
2445 return "rarfile";
2446 if(py7zr_support and compresscheck=="7zipfile" and py7zr.is_7zfile(infile)):
2447 return "7zipfile";
2448 if(hasattr(infile, "read") or hasattr(infile, "write")):
2449 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2450 else:
2451 try:
2452 if(compresscheck=="gzip"):
2453 try:
2454 import gzip;
2455 except ImportError:
2456 return False;
2457 catfp = gzip.GzipFile(infile, "rb");
2458 if(compresscheck=="bzip2"):
2459 try:
2460 import bz2;
2461 except ImportError:
2462 return False;
2463 catfp = bz2.BZ2File(infile, "rb");
2464 if(compresscheck=="lz4"):
2465 try:
2466 import lz4.frame;
2467 except ImportError:
2468 return False;
2469 catfp = lz4.frame.open(infile, "rb");
2470 if(compresscheck=="zstd"):
2471 try:
2472 import zstandard;
2473 except ImportError:
2474 return False;
2475 catfp = zstandard.open(infile, "rb");
2476 if(compresscheck=="lzma" or compresscheck=="xz"):
2477 try:
2478 import lzma;
2479 except ImportError:
2480 try:
2481 from backports import lzma
2482 except ImportError:
2483 return False;
2484 catfp = lzma.open(infile, "rb");
2485 except FileNotFoundError:
2486 return False;
2487 filetype = False;
2488 prefp = catfp.read(5);
2489 if(prefp==binascii.unhexlify("7573746172")):
2490 filetype = "tarfile";
2491 catfp.seek(0, 0);
2492 prefp = catfp.read(7);
2493 if(prefp==binascii.unhexlify("43617446696c65")):
2494 filetype = "catfile";
2495 catfp.seek(0, 0);
2496 prefp = catfp.read(formatspecs['format_len']);
2497 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2498 filetype = formatspecs['format_lower'];
2499 catfp.seek(0, 0);
2500 prefp = catfp.read(10);
2501 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2502 filetype = "tarfile";
2503 catfp.seek(0, 0);
2504 if(closefp):
2505 catfp.close();
2506 return filetype;
2507
2508 def GZipCompress(data, compresslevel=9):
2509 try:
2510 import gzip;
2511 except ImportError:
2512 return False;
2513 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2514 tmpfp.close();
2515 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2516 tmpfp.write(data);
2517 tmpfp.close();
2518 try:
2519 catfp = open(tmpfp.name, "rb");
2520 except FileNotFoundError:
2521 return False;
2522 catdata = catfp.read();
2523 catfp.close();
2524 return catdata;
2525
2526 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2527 formatspecs = FormatSpecsListToDict(formatspecs);
2528 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2529 return False;
2530 fp.seek(0, 0);
2531 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2532 compression = "auto";
2533 if(compression not in compressionlist and compression is None):
2534 compression = "auto";
2535 if(compression=="gzip"):
2536 try:
2537 import gzip;
2538 except ImportError:
2539 return False;
2540 catfp = BytesIO();
2541 if(compressionlevel is None):
2542 compressionlevel = 9;
2543 else:
2544 compressionlevel = int(compressionlevel);
2545 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2546 if(compression=="bzip2"):
2547 try:
2548 import bz2;
2549 except ImportError:
2550 return False;
2551 catfp = BytesIO();
2552 if(compressionlevel is None):
2553 compressionlevel = 9;
2554 else:
2555 compressionlevel = int(compressionlevel);
2556 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2557 if(compression=="lz4"):
2558 try:
2559 import lz4.frame;
2560 except ImportError:
2561 return False;
2562 catfp = BytesIO();
2563 if(compressionlevel is None):
2564 compressionlevel = 9;
2565 else:
2566 compressionlevel = int(compressionlevel);
2567 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2568 if(compression=="lzo" or compression=="lzop"):
2569 try:
2570 import lzo;
2571 except ImportError:
2572 return False;
2573 catfp = BytesIO();
2574 if(compressionlevel is None):
2575 compressionlevel = 9;
2576 else:
2577 compressionlevel = int(compressionlevel);
2578 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2579 if(compression=="zstd"):
2580 try:
2581 import zstandard;
2582 except ImportError:
2583 return False;
2584 catfp = BytesIO();
2585 if(compressionlevel is None):
2586 compressionlevel = 10;
2587 else:
2588 compressionlevel = int(compressionlevel);
2589 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2590 if(compression=="lzma"):
2591 try:
2592 import lzma;
2593 except ImportError:
2594 try:
2595 from backports import lzma
2596 except ImportError:
2597 return False;
2598 catfp = BytesIO();
2599 if(compressionlevel is None):
2600 compressionlevel = 9;
2601 else:
2602 compressionlevel = int(compressionlevel);
2603 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2604 if(compression=="xz"):
2605 try:
2606 import lzma;
2607 except ImportError:
2608 try:
2609 from backports import lzma
2610 except ImportError:
2611 return False;
2612 catfp = BytesIO();
2613 if(compressionlevel is None):
2614 compressionlevel = 9;
2615 else:
2616 compressionlevel = int(compressionlevel);
2617 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2618 if(compression=="auto" or compression is None):
2619 catfp = fp;
2620 catfp.seek(0, 0);
2621 return catfp;
2622
2623 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2624
2625 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2626 if(outfile is None):
2627 return False;
2628 fbasename = os.path.splitext(outfile)[0];
2629 fextname = os.path.splitext(outfile)[1];
2630 if(compressionlevel is None and fextname!=".zst"):
2631 compressionlevel = 9;
2632 elif(compressionlevel is None and fextname==".zst"):
2633 compressionlevel = 10;
2634 else:
2635 compressionlevel = int(compressionlevel);
2636 if(sys.version_info[0]==2):
2637 mode = "w";
2638 else:
2639 mode = "wb";
2640 try:
2641 if(fextname not in outextlistwd or not compressionenable):
2642 try:
2643 outfp = open(outfile, "wb", encoding="UTF-8");
2644 except (ValueError, TypeError) as e:
2645 outfp = open(outfile, "wb");
2646 elif(fextname==".gz"):
2647 try:
2648 import gzip;
2649 except ImportError:
2650 return False;
2651 try:
2652 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2653 except (ValueError, TypeError) as e:
2654 outfp = gzip.open(outfile, mode, compressionlevel);
2655 elif(fextname==".bz2"):
2656 try:
2657 import bz2;
2658 except ImportError:
2659 return False;
2660 try:
2661 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2662 except (ValueError, TypeError) as e:
2663 outfp = bz2.open(outfile, mode, compressionlevel);
2664 elif(fextname==".zst"):
2665 try:
2666 import zstandard;
2667 except ImportError:
2668 return False;
2669 try:
2670 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2671 except (ValueError, TypeError) as e:
2672 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2673 elif(fextname==".xz"):
2674 try:
2675 import lzma;
2676 except ImportError:
2677 try:
2678 from backports import lzma
2679 except ImportError:
2680 return False;
2681 try:
2682 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2683 except (ValueError, TypeError) as e:
2684 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2685 elif(fextname==".lz4"):
2686 try:
2687 import lz4.frame;
2688 except ImportError:
2689 return False;
2690 try:
2691 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2692 except (ValueError, TypeError) as e:
2693 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2694 elif(fextname==".lzo"):
2695 try:
2696 import lzo;
2697 except ImportError:
2698 return False;
2699 try:
2700 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2701 except (ValueError, TypeError) as e:
2702 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2703 elif(fextname==".lzma"):
2704 try:
2705 import lzma;
2706 except ImportError:
2707 try:
2708 from backports import lzma
2709 except ImportError:
2710 return False;
2711 try:
2712 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2713 except (ValueError, TypeError) as e:
2714 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2715 except FileNotFoundError:
2716 return False;
2717 return outfp;
2718
2719 def GetDevMajorMinor(fdev):
2720 retdev = [];
2721 if(hasattr(os, "minor")):
2722 retdev.append(os.minor(fdev));
2723 else:
2724 retdev.append(0);
2725 if(hasattr(os, "major")):
2726 retdev.append(os.major(fdev));
2727 else:
2728 retdev.append(0);
2729 return retdev;
2730
2731 def CheckSumSupport(checkfor, guaranteed=True):
2732 if(guaranteed):
2733 try:
2734 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2735 except AttributeError:
2736 hash_list = sorted(list(hashlib.algorithms));
2737 else:
2738 try:
2739 hash_list = sorted(list(hashlib.algorithms_available));
2740 except AttributeError:
2741 hash_list = sorted(list(hashlib.algorithms));
2742 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2743 if(checkfor in checklistout):
2744 return True;
2745 else:
2746 return False;
2747
2748 def CheckSumSupportAlt(checkfor, guaranteed=True):
2749 if(guaranteed):
2750 try:
2751 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2752 except AttributeError:
2753 hash_list = sorted(list(hashlib.algorithms));
2754 else:
2755 try:
2756 hash_list = sorted(list(hashlib.algorithms_available));
2757 except AttributeError:
2758 hash_list = sorted(list(hashlib.algorithms));
2759 checklistout = hash_list;
2760 if(checkfor in checklistout):
2761 return True;
2762 else:
2763 return False;
2764
2765 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2766 formatspecs = FormatSpecsListToDict(formatspecs);
2767 advancedlist = formatspecs['use_advanced_list'];
2768 altinode = formatspecs['use_alt_inode'];
2769 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2770 outfile = RemoveWindowsPath(outfile);
2771 checksumtype = checksumtype.lower();
2772 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2773 checksumtype="crc32";
2774 if(checksumtype=="none"):
2775 checksumtype = "";
2776 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2777 compression = "auto";
2778 if(compression not in compressionlist and compression is None):
2779 compression = "auto";
2780 if(verbose):
2781 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2782 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2783 if(os.path.exists(outfile)):
2784 try:
2785 os.unlink(outfile);
2786 except OSError as e:
2787 pass;
2788 if(outfile=="-"):
2789 verbose = False;
2790 catfp = BytesIO();
2791 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2792 catfp = outfile;
2793 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2794 catfp = BytesIO();
2795 else:
2796 fbasename = os.path.splitext(outfile)[0];
2797 fextname = os.path.splitext(outfile)[1];
2798 if(not compresswholefile and fextname in outextlistwd):
2799 compresswholefile = True;
2800 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2801 catver = formatspecs['format_ver'];
2802 fileheaderver = str(int(catver.replace(".", "")));
2803 infilelist = [];
2804 if(infiles=="-"):
2805 for line in sys.stdin:
2806 infilelist.append(line.strip());
2807 infilelist = list(filter(None, infilelist));
2808 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2809 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2810 return False;
2811 with UncompressFile(infiles, formatspecs, "r") as finfile:
2812 for line in finfile:
2813 infilelist.append(line.strip());
2814 infilelist = list(filter(None, infilelist));
2815 else:
2816 if(isinstance(infiles, (list, tuple, ))):
2817 infilelist = list(filter(None, infiles));
2818 elif(isinstance(infiles, (str, ))):
2819 infilelist = list(filter(None, [infiles]));
2820 if(advancedlist):
2821 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2822 else:
2823 GetDirList = ListDir(infilelist, followlink, False);
2824 if(not GetDirList):
2825 return False;
2826 curinode = 0;
2827 curfid = 0;
2828 inodelist = [];
2829 inodetofile = {};
2830 filetoinode = {};
2831 inodetocatinode = {};
2832 numfiles = int(len(GetDirList));
2833 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2834 for curfname in GetDirList:
2835 if(re.findall(r"^[.|/]", curfname)):
2836 fname = curfname;
2837 else:
2838 fname = "./"+curfname;
2839 if(verbose):
2840 VerbosePrintOut(fname);
2841 if(not followlink or followlink is None):
2842 fstatinfo = os.lstat(fname);
2843 else:
2844 fstatinfo = os.stat(fname);
2845 fpremode = fstatinfo.st_mode;
2846 finode = fstatinfo.st_ino;
2847 flinkcount = fstatinfo.st_nlink;
2848 ftype = 0;
2849 if(stat.S_ISREG(fpremode)):
2850 ftype = 0;
2851 elif(stat.S_ISLNK(fpremode)):
2852 ftype = 2;
2853 elif(stat.S_ISCHR(fpremode)):
2854 ftype = 3;
2855 elif(stat.S_ISBLK(fpremode)):
2856 ftype = 4;
2857 elif(stat.S_ISDIR(fpremode)):
2858 ftype = 5;
2859 elif(stat.S_ISFIFO(fpremode)):
2860 ftype = 6;
2861 elif(stat.S_ISSOCK(fpremode)):
2862 ftype = 8;
2863 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2864 ftype = 9;
2865 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2866 ftype = 10;
2867 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2868 ftype = 11;
2869 else:
2870 ftype = 0;
2871 flinkname = "";
2872 fcurfid = format(int(curfid), 'x').lower();
2873 if(not followlink and finode!=0):
2874 if(ftype!=1):
2875 if(finode in inodelist):
2876 ftype = 1;
2877 flinkname = inodetofile[finode];
2878 if(altinode):
2879 fcurinode = format(int(finode), 'x').lower();
2880 else:
2881 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2882 if(finode not in inodelist):
2883 inodelist.append(finode);
2884 inodetofile.update({finode: fname});
2885 inodetocatinode.update({finode: curinode});
2886 if(altinode):
2887 fcurinode = format(int(finode), 'x').lower();
2888 else:
2889 fcurinode = format(int(curinode), 'x').lower();
2890 curinode = curinode + 1;
2891 else:
2892 fcurinode = format(int(curinode), 'x').lower();
2893 curinode = curinode + 1;
2894 curfid = curfid + 1;
2895 if(ftype==2):
2896 flinkname = os.readlink(fname);
2897 fdev = fstatinfo.st_dev;
2898 getfdev = GetDevMajorMinor(fdev);
2899 fdev_minor = getfdev[0];
2900 fdev_major = getfdev[1];
2901 frdev = fstatinfo.st_dev;
2902 if(hasattr(fstatinfo, "st_rdev")):
2903 frdev = fstatinfo.st_rdev;
2904 else:
2905 frdev = fstatinfo.st_dev;
2906 getfrdev = GetDevMajorMinor(frdev);
2907 frdev_minor = getfrdev[0];
2908 frdev_major = getfrdev[1];
2909 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2910 fsize = format(int("0"), 'x').lower();
2911 elif(ftype==0 or ftype==7):
2912 fsize = format(int(fstatinfo.st_size), 'x').lower();
2913 else:
2914 fsize = format(int(fstatinfo.st_size)).lower();
2915 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2916 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2917 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2918 if(hasattr(fstatinfo, "st_birthtime")):
2919 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2920 else:
2921 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2922 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2923 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2924 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2925 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2926 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2927 funame = "";
2928 try:
2929 import pwd;
2930 try:
2931 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2932 funame = userinfo.pw_name;
2933 except KeyError:
2934 funame = "";
2935 except ImportError:
2936 funame = "";
2937 fgname = "";
2938 try:
2939 import grp;
2940 try:
2941 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2942 fgname = groupinfo.gr_name;
2943 except KeyError:
2944 fgname = "";
2945 except ImportError:
2946 fgname = "";
2947 fdev_minor = format(int(fdev_minor), 'x').lower();
2948 fdev_major = format(int(fdev_major), 'x').lower();
2949 frdev_minor = format(int(frdev_minor), 'x').lower();
2950 frdev_major = format(int(frdev_major), 'x').lower();
2951 finode = format(int(finode), 'x').lower();
2952 flinkcount = format(int(flinkcount), 'x').lower();
2953 if(hasattr(fstatinfo, "st_file_attributes")):
2954 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2955 else:
2956 fwinattributes = format(int(0), 'x').lower();
2957 fcompression = "";
2958 fcsize = format(int(0), 'x').lower();
2959 fcontents = BytesIO();
2960 if(ftype==0 or ftype==7):
2961 with open(fname, "rb") as fpc:
2962 shutil.copyfileobj(fpc, fcontents);
2963 if(not compresswholefile):
2964 fcontents.seek(0, 2);
2965 ucfsize = fcontents.tell();
2966 fcontents.seek(0, 0);
2967 if(compression=="auto"):
2968 ilsize = len(compressionlistalt);
2969 ilmin = 0;
2970 ilcsize = [];
2971 while(ilmin < ilsize):
2972 cfcontents = BytesIO();
2973 shutil.copyfileobj(fcontents, cfcontents);
2974 fcontents.seek(0, 0);
2975 cfcontents.seek(0, 0);
2976 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2977 if(cfcontents):
2978 cfcontents.seek(0, 2);
2979 ilcsize.append(cfcontents.tell());
2980 cfcontents.close();
2981 else:
2982 try:
2983 ilcsize.append(sys.maxint);
2984 except AttributeError:
2985 ilcsize.append(sys.maxsize);
2986 ilmin = ilmin + 1;
2987 ilcmin = ilcsize.index(min(ilcsize));
2988 compression = compressionlistalt[ilcmin];
2989 fcontents.seek(0, 0);
2990 cfcontents = BytesIO();
2991 shutil.copyfileobj(fcontents, cfcontents);
2992 cfcontents.seek(0, 0);
2993 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2994 cfcontents.seek(0, 2);
2995 cfsize = cfcontents.tell();
2996 if(ucfsize > cfsize):
2997 fcsize = format(int(cfsize), 'x').lower();
2998 fcompression = compression;
2999 fcontents.close();
3000 fcontents = cfcontents;
3001 if(fcompression=="none"):
3002 fcompression = "";
3003 if(followlink and (ftype==1 or ftype==2)):
3004 flstatinfo = os.stat(flinkname);
3005 with open(flinkname, "rb") as fpc:
3006 shutil.copyfileobj(fpc, fcontents);
3007 if(not compresswholefile):
3008 fcontents.seek(0, 2);
3009 ucfsize = fcontents.tell();
3010 fcontents.seek(0, 0);
3011 if(compression=="auto"):
3012 ilsize = len(compressionlistalt);
3013 ilmin = 0;
3014 ilcsize = [];
3015 while(ilmin < ilsize):
3016 cfcontents = BytesIO();
3017 shutil.copyfileobj(fcontents, cfcontents);
3018 fcontents.seek(0, 0);
3019 cfcontents.seek(0, 0);
3020 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3021 if(cfcontents):
3022 cfcontents.seek(0, 2);
3023 ilcsize.append(cfcontents.tell());
3024 cfcontents.close();
3025 else:
3026 try:
3027 ilcsize.append(sys.maxint);
3028 except AttributeError:
3029 ilcsize.append(sys.maxsize);
3030 ilmin = ilmin + 1;
3031 ilcmin = ilcsize.index(min(ilcsize));
3032 compression = compressionlistalt[ilcmin];
3033 fcontents.seek(0, 0);
3034 cfcontents = BytesIO();
3035 shutil.copyfileobj(fcontents, cfcontents);
3036 cfcontents.seek(0, 0);
3037 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3038 cfcontents.seek(0, 2);
3039 cfsize = cfcontents.tell();
3040 if(ucfsize > cfsize):
3041 fcsize = format(int(cfsize), 'x').lower();
3042 fcompression = compression;
3043 fcontents.close();
3044 fcontents = cfcontents;
3045 fcontents.seek(0, 0);
3046 ftypehex = format(ftype, 'x').lower();
3047 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3048 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3049 fcontents.close();
3050 if(numfiles>0):
3051 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3052 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3053 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3054 try:
3055 catfp.flush();
3056 os.fsync(catfp.fileno());
3057 except io.UnsupportedOperation:
3058 pass;
3059 except AttributeError:
3060 pass;
3061 except OSError as e:
3062 pass;
3063 if(outfile=="-"):
3064 catfp.seek(0, 0);
3065 if(hasattr(sys.stdout, "buffer")):
3066 shutil.copyfileobj(catfp, sys.stdout.buffer);
3067 else:
3068 shutil.copyfileobj(catfp, sys.stdout);
3069 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3070 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3071 catfp.seek(0, 0);
3072 upload_file_to_internet_file(catfp, outfile);
3073 if(returnfp):
3074 catfp.seek(0, 0);
3075 return catfp;
3076 else:
3077 catfp.close();
3078 return True;
3079
3080 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3081
3082 if(hasattr(shutil, "register_archive_format")):
3083 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3084 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3085 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3086
3087 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3088 formatspecs = FormatSpecsListToDict(formatspecs);
3089 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3090
3091 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3092
3093 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3094 formatspecs = FormatSpecsListToDict(formatspecs);
3095 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3096 outfile = RemoveWindowsPath(outfile);
3097 checksumtype = checksumtype.lower();
3098 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3099 checksumtype="crc32";
3100 if(checksumtype=="none"):
3101 checksumtype = "";
3102 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3103 compression = "auto";
3104 if(compression not in compressionlist and compression is None):
3105 compression = "auto";
3106 if(verbose):
3107 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3108 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3109 if(os.path.exists(outfile)):
3110 try:
3111 os.unlink(outfile);
3112 except OSError as e:
3113 pass;
3114 if(outfile=="-"):
3115 verbose = False;
3116 catfp = BytesIO();
3117 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3118 catfp = outfile;
3119 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3120 catfp = BytesIO();
3121 else:
3122 fbasename = os.path.splitext(outfile)[0];
3123 fextname = os.path.splitext(outfile)[1];
3124 if(not compresswholefile and fextname in outextlistwd):
3125 compresswholefile = True;
3126 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3127 catver = formatspecs['format_ver'];
3128 fileheaderver = str(int(catver.replace(".", "")));
3129 curinode = 0;
3130 curfid = 0;
3131 inodelist = [];
3132 inodetofile = {};
3133 filetoinode = {};
3134 inodetocatinode = {};
3135 if(infile=="-"):
3136 infile = BytesIO();
3137 if(hasattr(sys.stdin, "buffer")):
3138 shutil.copyfileobj(sys.stdin.buffer, infile);
3139 else:
3140 shutil.copyfileobj(sys.stdin, infile);
3141 infile.seek(0, 0);
3142 if(not infile):
3143 return False;
3144 infile.seek(0, 0);
3145 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3146 infile = download_file_from_internet_file(infile);
3147 infile.seek(0, 0);
3148 if(not infile):
3149 return False;
3150 infile.seek(0, 0);
3151 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3152 return False;
3153 elif(os.path.exists(infile) and os.path.isfile(infile)):
3154 try:
3155 if(not tarfile.TarFileCheck(infile)):
3156 return False;
3157 except AttributeError:
3158 if(not TarFileCheck(infile)):
3159 return False;
3160 else:
3161 return False;
3162 try:
3163 if(hasattr(infile, "read") or hasattr(infile, "write")):
3164 tarfp = tarfile.open(fileobj=infile, mode="r");
3165 else:
3166 tarfp = tarfile.open(infile, "r");
3167 except FileNotFoundError:
3168 return False;
3169 numfiles = int(len(tarfp.getmembers()));
3170 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3171 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3172 if(re.findall(r"^[.|/]", member.name)):
3173 fname = member.name;
3174 else:
3175 fname = "./"+member.name;
3176 if(verbose):
3177 VerbosePrintOut(fname);
3178 fpremode = member.mode;
3179 ffullmode = member.mode;
3180 flinkcount = 0;
3181 ftype = 0;
3182 if(member.isreg()):
3183 ffullmode = member.mode + stat.S_IFREG;
3184 ftype = 0;
3185 elif(member.isdev()):
3186 ffullmode = member.mode;
3187 ftype = 7;
3188 elif(member.islnk()):
3189 ffullmode = member.mode + stat.S_IFREG;
3190 ftype = 1;
3191 elif(member.issym()):
3192 ffullmode = member.mode + stat.S_IFLNK;
3193 ftype = 2;
3194 elif(member.ischr()):
3195 ffullmode = member.mode + stat.S_IFCHR;
3196 ftype = 3;
3197 elif(member.isblk()):
3198 ffullmode = member.mode + stat.S_IFBLK;
3199 ftype = 4;
3200 elif(member.isdir()):
3201 ffullmode = member.mode + stat.S_IFDIR;
3202 ftype = 5;
3203 elif(member.isfifo()):
3204 ffullmode = member.mode + stat.S_IFIFO;
3205 ftype = 6;
3206 elif(member.issparse()):
3207 ffullmode = member.mode;
3208 ftype = 12;
3209 else:
3210 ffullmode = member.mode;
3211 ftype = 0;
3212 flinkname = "";
3213 fcurfid = format(int(curfid), 'x').lower();
3214 fcurinode = format(int(curfid), 'x').lower();
3215 curfid = curfid + 1;
3216 if(ftype==2):
3217 flinkname = member.linkname;
3218 fdev_minor = format(int(member.devminor), 'x').lower();
3219 fdev_major = format(int(member.devmajor), 'x').lower();
3220 frdev_minor = format(int(member.devminor), 'x').lower();
3221 frdev_major = format(int(member.devmajor), 'x').lower();
3222 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3223 fsize = format(int("0"), 'x').lower();
3224 elif(ftype==0 or ftype==7):
3225 fsize = format(int(member.size), 'x').lower();
3226 else:
3227 fsize = format(int(member.size), 'x').lower();
3228 fatime = format(int(member.mtime), 'x').lower();
3229 fmtime = format(int(member.mtime), 'x').lower();
3230 fctime = format(int(member.mtime), 'x').lower();
3231 fbtime = format(int(member.mtime), 'x').lower();
3232 fmode = format(int(ffullmode), 'x').lower();
3233 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3234 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3235 fuid = format(int(member.uid), 'x').lower();
3236 fgid = format(int(member.gid), 'x').lower();
3237 funame = member.uname;
3238 fgname = member.gname;
3239 flinkcount = format(int(flinkcount), 'x').lower();
3240 fwinattributes = format(int(0), 'x').lower();
3241 fcompression = "";
3242 fcsize = format(int(0), 'x').lower();
3243 fcontents = BytesIO();
3244 if(ftype==0 or ftype==7):
3245 with tarfp.extractfile(member) as fpc:
3246 shutil.copyfileobj(fpc, fcontents);
3247 if(not compresswholefile):
3248 fcontents.seek(0, 2);
3249 ucfsize = fcontents.tell();
3250 fcontents.seek(0, 0);
3251 if(compression=="auto"):
3252 ilsize = len(compressionlistalt);
3253 ilmin = 0;
3254 ilcsize = [];
3255 while(ilmin < ilsize):
3256 cfcontents = BytesIO();
3257 shutil.copyfileobj(fcontents, cfcontents);
3258 fcontents.seek(0, 0);
3259 cfcontents.seek(0, 0);
3260 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3261 if(cfcontents):
3262 cfcontents.seek(0, 2);
3263 ilcsize.append(cfcontents.tell());
3264 cfcontents.close();
3265 else:
3266 try:
3267 ilcsize.append(sys.maxint);
3268 except AttributeError:
3269 ilcsize.append(sys.maxsize);
3270 ilmin = ilmin + 1;
3271 ilcmin = ilcsize.index(min(ilcsize));
3272 compression = compressionlistalt[ilcmin];
3273 fcontents.seek(0, 0);
3274 cfcontents = BytesIO();
3275 shutil.copyfileobj(fcontents, cfcontents);
3276 cfcontents.seek(0, 0);
3277 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3278 cfcontents.seek(0, 2);
3279 cfsize = cfcontents.tell();
3280 if(ucfsize > cfsize):
3281 fcsize = format(int(cfsize), 'x').lower();
3282 fcompression = compression;
3283 fcontents.close();
3284 fcontents = cfcontents;
3285 if(fcompression=="none"):
3286 fcompression = "";
3287 fcontents.seek(0, 0);
3288 ftypehex = format(ftype, 'x').lower();
3289 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3290 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3291 fcontents.close();
3292 if(numfiles>0):
3293 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3294 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3295 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3296 try:
3297 catfp.flush();
3298 os.fsync(catfp.fileno());
3299 except io.UnsupportedOperation:
3300 pass;
3301 except AttributeError:
3302 pass;
3303 except OSError as e:
3304 pass;
3305 if(outfile=="-"):
3306 catfp.seek(0, 0);
3307 if(hasattr(sys.stdout, "buffer")):
3308 shutil.copyfileobj(catfp, sys.stdout.buffer);
3309 else:
3310 shutil.copyfileobj(catfp, sys.stdout);
3311 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3312 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3313 catfp.seek(0, 0);
3314 upload_file_to_internet_file(catfp, outfile);
3315 if(returnfp):
3316 catfp.seek(0, 0);
3317 return catfp;
3318 else:
3319 catfp.close();
3320 return True;
3321
3322 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3323
3324 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3325 formatspecs = FormatSpecsListToDict(formatspecs);
3326 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3327 outfile = RemoveWindowsPath(outfile);
3328 checksumtype = checksumtype.lower();
3329 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3330 checksumtype="crc32";
3331 if(checksumtype=="none"):
3332 checksumtype = "";
3333 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3334 compression = "auto";
3335 if(compression not in compressionlist and compression is None):
3336 compression = "auto";
3337 if(verbose):
3338 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3339 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3340 if(os.path.exists(outfile)):
3341 try:
3342 os.unlink(outfile);
3343 except OSError as e:
3344 pass;
3345 if(outfile=="-"):
3346 verbose = False;
3347 catfp = BytesIO();
3348 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3349 catfp = outfile;
3350 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3351 catfp = BytesIO();
3352 else:
3353 fbasename = os.path.splitext(outfile)[0];
3354 fextname = os.path.splitext(outfile)[1];
3355 if(not compresswholefile and fextname in outextlistwd):
3356 compresswholefile = True;
3357 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3358 catver = formatspecs['format_ver'];
3359 fileheaderver = str(int(catver.replace(".", "")));
3360 curinode = 0;
3361 curfid = 0;
3362 inodelist = [];
3363 inodetofile = {};
3364 filetoinode = {};
3365 inodetocatinode = {};
3366 if(infile=="-"):
3367 infile = BytesIO();
3368 if(hasattr(sys.stdin, "buffer")):
3369 shutil.copyfileobj(sys.stdin.buffer, infile);
3370 else:
3371 shutil.copyfileobj(sys.stdin, infile);
3372 infile.seek(0, 0);
3373 if(not infile):
3374 return False;
3375 infile.seek(0, 0);
3376 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3377 infile = download_file_from_internet_file(infile);
3378 infile.seek(0, 0);
3379 if(not infile):
3380 return False;
3381 infile.seek(0, 0);
3382 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3383 return False;
3384 else:
3385 return False;
3386 if(not zipfile.is_zipfile(infile)):
3387 return False;
3388 try:
3389 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3390 except FileNotFoundError:
3391 return False;
3392 ziptest = zipfp.testzip();
3393 if(ziptest):
3394 VerbosePrintOut("Bad file found!");
3395 numfiles = int(len(zipfp.infolist()));
3396 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3397 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3398 if(re.findall(r"^[.|/]", member.filename)):
3399 fname = member.filename;
3400 else:
3401 fname = "./"+member.filename;
3402 zipinfo = zipfp.getinfo(member.filename);
3403 if(verbose):
3404 VerbosePrintOut(fname);
3405 if(not member.is_dir()):
3406 fpremode = int(stat.S_IFREG + 438);
3407 elif(member.is_dir()):
3408 fpremode = int(stat.S_IFDIR + 511);
3409 flinkcount = 0;
3410 ftype = 0;
3411 if(not member.is_dir()):
3412 ftype = 0;
3413 elif(member.is_dir()):
3414 ftype = 5;
3415 flinkname = "";
3416 fcurfid = format(int(curfid), 'x').lower();
3417 fcurinode = format(int(curfid), 'x').lower();
3418 curfid = curfid + 1;
3419 fdev_minor = format(int(0), 'x').lower();
3420 fdev_major = format(int(0), 'x').lower();
3421 frdev_minor = format(int(0), 'x').lower();
3422 frdev_major = format(int(0), 'x').lower();
3423 if(ftype==5):
3424 fsize = format(int("0"), 'x').lower();
3425 elif(ftype==0):
3426 fsize = format(int(member.file_size), 'x').lower();
3427 else:
3428 fsize = format(int(member.file_size), 'x').lower();
3429 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3430 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3431 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3432 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3433 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3434 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3435 if(not member.is_dir()):
3436 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3437 fchmode = stat.S_IMODE(fmode);
3438 ftypemod = stat.S_IFMT(fmode);
3439 elif(member.is_dir()):
3440 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3441 fchmode = stat.S_IMODE(fmode);
3442 ftypemod = stat.S_IFMT(fmode);
3443 elif(zipinfo.create_system==3):
3444 fwinattributes = format(int(0), 'x').lower();
3445 fmode = format(int(zipinfo.external_attr), 'x').lower();
3446 fchmode = stat.S_IMODE(fmode);
3447 ftypemod = stat.S_IFMT(fmode);
3448 else:
3449 fwinattributes = format(int(0), 'x').lower();
3450 if(not member.is_dir()):
3451 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3452 fchmode = stat.S_IMODE(fmode);
3453 ftypemod = stat.S_IFMT(fmode);
3454 elif(member.is_dir()):
3455 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3456 fchmode = stat.S_IMODE(fmode);
3457 ftypemod = stat.S_IFMT(fmode);
3458 fcompression = "";
3459 fcsize = format(int(0), 'x').lower();
3460 try:
3461 fuid = format(int(os.getuid()), 'x').lower();
3462 except AttributeError:
3463 fuid = format(int(0), 'x').lower();
3464 except KeyError:
3465 fuid = format(int(0), 'x').lower();
3466 try:
3467 fgid = format(int(os.getgid()), 'x').lower();
3468 except AttributeError:
3469 fgid = format(int(0), 'x').lower();
3470 except KeyError:
3471 fgid = format(int(0), 'x').lower();
3472 try:
3473 import pwd;
3474 try:
3475 userinfo = pwd.getpwuid(os.getuid());
3476 funame = userinfo.pw_name;
3477 except KeyError:
3478 funame = "";
3479 except AttributeError:
3480 funame = "";
3481 except ImportError:
3482 funame = "";
3483 fgname = "";
3484 try:
3485 import grp;
3486 try:
3487 groupinfo = grp.getgrgid(os.getgid());
3488 fgname = groupinfo.gr_name;
3489 except KeyError:
3490 fgname = "";
3491 except AttributeError:
3492 fgname = "";
3493 except ImportError:
3494 fgname = "";
3495 fcontents = BytesIO();
3496 if(ftype==0):
3497 fcontents.write(zipfp.read(member.filename));
3498 if(not compresswholefile):
3499 fcontents.seek(0, 2);
3500 ucfsize = fcontents.tell();
3501 fcontents.seek(0, 0);
3502 if(compression=="auto"):
3503 ilsize = len(compressionlistalt);
3504 ilmin = 0;
3505 ilcsize = [];
3506 while(ilmin < ilsize):
3507 cfcontents = BytesIO();
3508 shutil.copyfileobj(fcontents, cfcontents);
3509 fcontents.seek(0, 0);
3510 cfcontents.seek(0, 0);
3511 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3512 cfcontents.seek(0, 2);
3513 ilcsize.append(cfcontents.tell());
3514 cfcontents.close();
3515 ilmin = ilmin + 1;
3516 ilcmin = ilcsize.index(min(ilcsize));
3517 compression = compressionlistalt[ilcmin];
3518 fcontents.seek(0, 0);
3519 cfcontents = BytesIO();
3520 shutil.copyfileobj(fcontents, cfcontents);
3521 cfcontents.seek(0, 0);
3522 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3523 cfcontents.seek(0, 2);
3524 cfsize = cfcontents.tell();
3525 if(ucfsize > cfsize):
3526 fcsize = format(int(cfsize), 'x').lower();
3527 fcompression = compression;
3528 fcontents.close();
3529 fcontents = cfcontents;
3530 if(fcompression=="none"):
3531 fcompression = "";
3532 fcontents.seek(0, 0);
3533 ftypehex = format(ftype, 'x').lower();
3534 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3535 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3536 fcontents.close();
3537 if(numfiles>0):
3538 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3539 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3540 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3541 try:
3542 catfp.flush();
3543 os.fsync(catfp.fileno());
3544 except io.UnsupportedOperation:
3545 pass;
3546 except AttributeError:
3547 pass;
3548 except OSError as e:
3549 pass;
3550 if(outfile=="-"):
3551 catfp.seek(0, 0);
3552 if(hasattr(sys.stdout, "buffer")):
3553 shutil.copyfileobj(catfp, sys.stdout.buffer);
3554 else:
3555 shutil.copyfileobj(catfp, sys.stdout);
3556 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3557 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3558 catfp.seek(0, 0);
3559 upload_file_to_internet_file(catfp, outfile);
3560 if(returnfp):
3561 catfp.seek(0, 0);
3562 return catfp;
3563 else:
3564 catfp.close();
3565 return True;
3566
3567 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3568
3569 if(not rarfile_support):
3570 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3571 return False
3572
3573 if(rarfile_support):
3574 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3575 formatspecs = FormatSpecsListToDict(formatspecs);
3576 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3577 outfile = RemoveWindowsPath(outfile);
3578 checksumtype = checksumtype.lower();
3579 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3580 checksumtype="crc32";
3581 if(checksumtype=="none"):
3582 checksumtype = "";
3583 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3584 compression = "auto";
3585 if(compression not in compressionlist and compression is None):
3586 compression = "auto";
3587 if(verbose):
3588 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3589 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3590 if(os.path.exists(outfile)):
3591 try:
3592 os.unlink(outfile);
3593 except OSError as e:
3594 pass;
3595 if(outfile=="-"):
3596 verbose = False;
3597 catfp = BytesIO();
3598 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3599 catfp = outfile;
3600 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3601 catfp = BytesIO();
3602 else:
3603 fbasename = os.path.splitext(outfile)[0];
3604 fextname = os.path.splitext(outfile)[1];
3605 if(not compresswholefile and fextname in outextlistwd):
3606 compresswholefile = True;
3607 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3608 catver = formatspecs['format_ver'];
3609 fileheaderver = str(int(catver.replace(".", "")));
3610 curinode = 0;
3611 curfid = 0;
3612 inodelist = [];
3613 inodetofile = {};
3614 filetoinode = {};
3615 inodetocatinode = {};
3616 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3617 return False;
3618 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3619 return False;
3620 rarfp = rarfile.RarFile(infile, "r");
3621 rartest = rarfp.testrar();
3622 if(rartest):
3623 VerbosePrintOut("Bad file found!");
3624 numfiles = int(len(rarfp.infolist()));
3625 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3626 try:
3627 catfp.flush();
3628 os.fsync(catfp.fileno());
3629 except io.UnsupportedOperation:
3630 pass;
3631 except AttributeError:
3632 pass;
3633 except OSError as e:
3634 pass;
3635 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3636 is_unix = False;
3637 is_windows = False;
3638 if(member.host_os==rarfile.RAR_OS_UNIX):
3639 is_windows = False;
3640 try:
3641 member.external_attr
3642 is_unix = True;
3643 except AttributeError:
3644 is_unix = False;
3645 elif(member.host_os==rarfile.RAR_OS_WIN32):
3646 is_unix = False;
3647 try:
3648 member.external_attr
3649 is_windows = True;
3650 except AttributeError:
3651 is_windows = False;
3652 else:
3653 is_unix = False;
3654 is_windows = False;
3655 if(re.findall(r"^[.|/]", member.filename)):
3656 fname = member.filename;
3657 else:
3658 fname = "./"+member.filename;
3659 rarinfo = rarfp.getinfo(member.filename);
3660 if(verbose):
3661 VerbosePrintOut(fname);
3662 if(is_unix and member.external_attr !=0):
3663 fpremode = int(member.external_attr);
3664 elif(member.is_file()):
3665 fpremode = int(stat.S_IFREG + 438);
3666 elif(member.is_symlink()):
3667 fpremode = int(stat.S_IFLNK + 438);
3668 elif(member.is_dir()):
3669 fpremode = int(stat.S_IFDIR + 511);
3670 if(is_windows and member.external_attr !=0):
3671 fwinattributes = format(int(member.external_attr), 'x').lower();
3672 else:
3673 fwinattributes = format(int(0), 'x').lower();
3674 fcompression = "";
3675 fcsize = format(int(0), 'x').lower();
3676 flinkcount = 0;
3677 ftype = 0;
3678 if(member.is_file()):
3679 ftype = 0;
3680 elif(member.is_symlink()):
3681 ftype = 2;
3682 elif(member.is_dir()):
3683 ftype = 5;
3684 flinkname = "";
3685 if(ftype==2):
3686 flinkname = rarfp.read(member.filename).decode("UTF-8");
3687 fcurfid = format(int(curfid), 'x').lower();
3688 fcurinode = format(int(curfid), 'x').lower();
3689 curfid = curfid + 1;
3690 fdev_minor = format(int(0), 'x').lower();
3691 fdev_major = format(int(0), 'x').lower();
3692 frdev_minor = format(int(0), 'x').lower();
3693 frdev_major = format(int(0), 'x').lower();
3694 if(ftype==5):
3695 fsize = format(int("0"), 'x').lower();
3696 elif(ftype==0):
3697 fsize = format(int(member.file_size), 'x').lower();
3698 else:
3699 fsize = format(int(member.file_size), 'x').lower();
3700 try:
3701 if(member.atime):
3702 fatime = format(int(member.atime.timestamp()), 'x').lower();
3703 else:
3704 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3705 except AttributeError:
3706 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3707 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3708 try:
3709 if(member.ctime):
3710 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3711 else:
3712 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3713 except AttributeError:
3714 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3715 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3716 if(is_unix and member.external_attr !=0):
3717 fmode = format(int(member.external_attr), 'x').lower();
3718 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3719 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3720 elif(member.is_file()):
3721 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3722 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3723 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3724 elif(member.is_symlink()):
3725 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3726 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3727 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3728 elif(member.is_dir()):
3729 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3730 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3731 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3732 try:
3733 fuid = format(int(os.getuid()), 'x').lower();
3734 except AttributeError:
3735 fuid = format(int(0), 'x').lower();
3736 except KeyError:
3737 fuid = format(int(0), 'x').lower();
3738 try:
3739 fgid = format(int(os.getgid()), 'x').lower();
3740 except AttributeError:
3741 fgid = format(int(0), 'x').lower();
3742 except KeyError:
3743 fgid = format(int(0), 'x').lower();
3744 try:
3745 import pwd;
3746 try:
3747 userinfo = pwd.getpwuid(os.getuid());
3748 funame = userinfo.pw_name;
3749 except KeyError:
3750 funame = "";
3751 except AttributeError:
3752 funame = "";
3753 except ImportError:
3754 funame = "";
3755 fgname = "";
3756 try:
3757 import grp;
3758 try:
3759 groupinfo = grp.getgrgid(os.getgid());
3760 fgname = groupinfo.gr_name;
3761 except KeyError:
3762 fgname = "";
3763 except AttributeError:
3764 fgname = "";
3765 except ImportError:
3766 fgname = "";
3767 fcontents = BytesIO();
3768 if(ftype==0):
3769 fcontents.write(rarfp.read(member.filename));
3770 if(not compresswholefile):
3771 fcontents.seek(0, 2);
3772 ucfsize = fcontents.tell();
3773 fcontents.seek(0, 0);
3774 if(compression=="auto"):
3775 ilsize = len(compressionlistalt);
3776 ilmin = 0;
3777 ilcsize = [];
3778 while(ilmin < ilsize):
3779 cfcontents = BytesIO();
3780 shutil.copyfileobj(fcontents, cfcontents);
3781 fcontents.seek(0, 0);
3782 cfcontents.seek(0, 0);
3783 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3784 if(cfcontents):
3785 cfcontents.seek(0, 2);
3786 ilcsize.append(cfcontents.tell());
3787 cfcontents.close();
3788 else:
3789 try:
3790 ilcsize.append(sys.maxint);
3791 except AttributeError:
3792 ilcsize.append(sys.maxsize);
3793 ilmin = ilmin + 1;
3794 ilcmin = ilcsize.index(min(ilcsize));
3795 compression = compressionlistalt[ilcmin];
3796 fcontents.seek(0, 0);
3797 cfcontents = BytesIO();
3798 shutil.copyfileobj(fcontents, cfcontents);
3799 cfcontents.seek(0, 0);
3800 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3801 cfcontents.seek(0, 2);
3802 cfsize = cfcontents.tell();
3803 if(ucfsize > cfsize):
3804 fcsize = format(int(cfsize), 'x').lower();
3805 fcompression = compression;
3806 fcontents.close();
3807 fcontents = cfcontents;
3808 if(fcompression=="none"):
3809 fcompression = "";
3810 fcontents.seek(0, 0);
3811 ftypehex = format(ftype, 'x').lower();
3812 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3813 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3814 fcontents.close();
3815 if(numfiles>0):
3816 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3817 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3818 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3819 try:
3820 catfp.flush();
3821 os.fsync(catfp.fileno());
3822 except io.UnsupportedOperation:
3823 pass;
3824 except AttributeError:
3825 pass;
3826 except OSError as e:
3827 pass;
3828 if(outfile=="-"):
3829 catfp.seek(0, 0)
3830 if(hasattr(sys.stdout, "buffer")):
3831 shutil.copyfileobj(catfp, sys.stdout.buffer);
3832 else:
3833 shutil.copyfileobj(catfp, sys.stdout);
3834 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3835 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3836 catfp.seek(0, 0);
3837 upload_file_to_internet_file(catfp, outfile);
3838 if(returnfp):
3839 catfp.seek(0, 0)
3840 return catfp
3841 else:
3842 catfp.close()
3843 return True;
3844
3845 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3846
3847 if(not py7zr_support):
3848 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3849 return False
3850
3851 if(py7zr_support):
3852 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3853 formatspecs = FormatSpecsListToDict(formatspecs);
3854 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3855 outfile = RemoveWindowsPath(outfile);
3856 checksumtype = checksumtype.lower();
3857 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3858 checksumtype="crc32";
3859 if(checksumtype=="none"):
3860 checksumtype = "";
3861 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3862 compression = "auto";
3863 if(compression not in compressionlist and compression is None):
3864 compression = "auto";
3865 if(verbose):
3866 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3867 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3868 if(os.path.exists(outfile)):
3869 try:
3870 os.unlink(outfile);
3871 except OSError as e:
3872 pass;
3873 if(outfile=="-"):
3874 verbose = False;
3875 catfp = BytesIO();
3876 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3877 catfp = outfile;
3878 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3879 catfp = BytesIO();
3880 else:
3881 fbasename = os.path.splitext(outfile)[0];
3882 fextname = os.path.splitext(outfile)[1];
3883 if(not compresswholefile and fextname in outextlistwd):
3884 compresswholefile = True;
3885 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3886 catver = formatspecs['format_ver'];
3887 fileheaderver = str(int(catver.replace(".", "")));
3888 curinode = 0;
3889 curfid = 0;
3890 inodelist = [];
3891 inodetofile = {};
3892 filetoinode = {};
3893 inodetocatinode = {};
3894 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3895 return False;
3896 szpfp = py7zr.SevenZipFile(infile, mode="r");
3897 file_content = szpfp.readall();
3898 #sztest = szpfp.testzip();
3899 sztestalt = szpfp.test();
3900 if(sztestalt):
3901 VerbosePrintOut("Bad file found!");
3902 numfiles = int(len(szpfp.list()));
3903 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3904 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3905 if(re.findall(r"^[.|/]", member.filename)):
3906 fname = member.filename;
3907 else:
3908 fname = "./"+member.filename;
3909 if(verbose):
3910 VerbosePrintOut(fname);
3911 if(not member.is_directory):
3912 fpremode = int(stat.S_IFREG + 438);
3913 elif(member.is_directory):
3914 fpremode = int(stat.S_IFDIR + 511);
3915 fwinattributes = format(int(0), 'x').lower();
3916 fcompression = "";
3917 fcsize = format(int(0), 'x').lower();
3918 flinkcount = 0;
3919 ftype = 0;
3920 if(member.is_directory):
3921 ftype = 5;
3922 else:
3923 ftype = 0;
3924 flinkname = "";
3925 fcurfid = format(int(curfid), 'x').lower();
3926 fcurinode = format(int(curfid), 'x').lower();
3927 curfid = curfid + 1;
3928 fdev_minor = format(int(0), 'x').lower();
3929 fdev_major = format(int(0), 'x').lower();
3930 frdev_minor = format(int(0), 'x').lower();
3931 frdev_major = format(int(0), 'x').lower();
3932 if(ftype==5):
3933 fsize = format(int("0"), 'x').lower();
3934 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3935 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3936 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3937 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3938 if(member.is_directory):
3939 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3940 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3941 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3942 else:
3943 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3944 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3945 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3946 try:
3947 fuid = format(int(os.getuid()), 'x').lower();
3948 except AttributeError:
3949 fuid = format(int(0), 'x').lower();
3950 except KeyError:
3951 fuid = format(int(0), 'x').lower();
3952 try:
3953 fgid = format(int(os.getgid()), 'x').lower();
3954 except AttributeError:
3955 fgid = format(int(0), 'x').lower();
3956 except KeyError:
3957 fgid = format(int(0), 'x').lower();
3958 try:
3959 import pwd;
3960 try:
3961 userinfo = pwd.getpwuid(os.getuid());
3962 funame = userinfo.pw_name;
3963 except KeyError:
3964 funame = "";
3965 except AttributeError:
3966 funame = "";
3967 except ImportError:
3968 funame = "";
3969 fgname = "";
3970 try:
3971 import grp;
3972 try:
3973 groupinfo = grp.getgrgid(os.getgid());
3974 fgname = groupinfo.gr_name;
3975 except KeyError:
3976 fgname = "";
3977 except AttributeError:
3978 fgname = "";
3979 except ImportError:
3980 fgname = "";
3981 fcontents = BytesIO();
3982 if(ftype==0):
3983 fcontents.write(file_content[member.filename].read());
3984 fsize = format(fcontents.tell(), 'x').lower();
3985 file_content[member.filename].close();
3986 if(not compresswholefile):
3987 fcontents.seek(0, 2);
3988 ucfsize = fcontents.tell();
3989 fcontents.seek(0, 0);
3990 if(compression=="auto"):
3991 ilsize = len(compressionlistalt);
3992 ilmin = 0;
3993 ilcsize = [];
3994 while(ilmin < ilsize):
3995 cfcontents = BytesIO();
3996 shutil.copyfileobj(fcontents, cfcontents);
3997 fcontents.seek(0, 0);
3998 cfcontents.seek(0, 0);
3999 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
4000 if(cfcontents):
4001 cfcontents.seek(0, 2);
4002 ilcsize.append(cfcontents.tell());
4003 cfcontents.close();
4004 else:
4005 try:
4006 ilcsize.append(sys.maxint);
4007 except AttributeError:
4008 ilcsize.append(sys.maxsize);
4009 ilmin = ilmin + 1;
4010 ilcmin = ilcsize.index(min(ilcsize));
4011 compression = compressionlistalt[ilcmin];
4012 fcontents.seek(0, 0);
4013 cfcontents = BytesIO();
4014 shutil.copyfileobj(fcontents, cfcontents);
4015 cfcontents.seek(0, 0);
4016 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4017 cfcontents.seek(0, 2);
4018 cfsize = cfcontents.tell();
4019 if(ucfsize > cfsize):
4020 fcsize = format(int(cfsize), 'x').lower();
4021 fcompression = compression;
4022 fcontents.close();
4023 fcontents = cfcontents;
4024 if(fcompression=="none"):
4025 fcompression = "";
4026 fcontents.seek(0, 0);
4027 ftypehex = format(ftype, 'x').lower();
4028 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4029 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4030 fcontents.close();
4031 if(numfiles>0):
4032 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4033 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4034 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4035 try:
4036 catfp.flush();
4037 os.fsync(catfp.fileno());
4038 except io.UnsupportedOperation:
4039 pass;
4040 except AttributeError:
4041 pass;
4042 except OSError as e:
4043 pass;
4044 if(outfile=="-"):
4045 catfp.seek(0, 0);
4046 if(hasattr(sys.stdout, "buffer")):
4047 shutil.copyfileobj(catfp, sys.stdout.buffer);
4048 else:
4049 shutil.copyfileobj(catfp, sys.stdout);
4050 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4051 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4052 catfp.seek(0, 0);
4053 upload_file_to_internet_file(catfp, outfile);
4054 if(returnfp):
4055 catfp.seek(0, 0)
4056 return catfp
4057 else:
4058 catfp.close()
4059 return True;
4060
4061 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4062
4063 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4064 formatspecs = FormatSpecsListToDict(formatspecs);
4065 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4066 if(verbose):
4067 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4068 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4069 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4070 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4071 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4072 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4073 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4074 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4075 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4076 elif(checkcompressfile=="catfile"):
4077 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4078 else:
4079 return False;
4080 return False;
4081
4082 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4083
4084 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4085 formatspecs = FormatSpecsListToDict(formatspecs);
4086 if(hasattr(infile, "read") or hasattr(infile, "write")):
4087 catfp = infile;
4088 catfp.seek(0, 0);
4089 catfp = UncompressArchiveFile(catfp, formatspecs);
4090 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4091 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4092 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4093 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4094 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4095 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4096 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4097 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4098 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4099 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4100 return False;
4101 if(not catfp):
4102 return False;
4103 catfp.seek(0, 0);
4104 elif(infile=="-"):
4105 catfp = BytesIO();
4106 if(hasattr(sys.stdin, "buffer")):
4107 shutil.copyfileobj(sys.stdin.buffer, catfp);
4108 else:
4109 shutil.copyfileobj(sys.stdin, catfp);
4110 catfp.seek(0, 0);
4111 catfp = UncompressArchiveFile(catfp, formatspecs);
4112 if(not catfp):
4113 return False;
4114 catfp.seek(0, 0);
4115 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4116 catfp = download_file_from_internet_file(infile);
4117 catfp.seek(0, 0);
4118 catfp = UncompressArchiveFile(catfp, formatspecs);
4119 if(not catfp):
4120 return False;
4121 catfp.seek(0, 0);
4122 else:
4123 infile = RemoveWindowsPath(infile);
4124 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4125 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4126 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4127 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4128 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4129 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4130 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4131 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4132 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4133 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4134 return False;
4135 compresscheck = CheckCompressionType(infile, formatspecs, True);
4136 if(not compresscheck):
4137 fextname = os.path.splitext(infile)[1];
4138 if(fextname==".gz"):
4139 compresscheck = "gzip";
4140 elif(fextname==".bz2"):
4141 compresscheck = "bzip2";
4142 elif(fextname==".zst"):
4143 compresscheck = "zstd";
4144 elif(fextname==".lz4" or fextname==".clz4"):
4145 compresscheck = "lz4";
4146 elif(fextname==".lzo" or fextname==".lzop"):
4147 compresscheck = "lzo";
4148 elif(fextname==".lzma" or fextname==".xz"):
4149 compresscheck = "lzma";
4150 else:
4151 return False;
4152 if(not compresscheck):
4153 return False;
4154 catfp = UncompressFile(infile, formatspecs, "rb");
4155 '''
4156 try:
4157 catfp.seek(0, 2);
4158 except OSError:
4159 SeekToEndOfFile(catfp);
4160 except ValueError:
4161 SeekToEndOfFile(catfp);
4162 CatSize = catfp.tell();
4163 CatSizeEnd = CatSize;
4164 '''
4165 try:
4166 catfp.seek(0, 0);
4167 except OSError:
4168 return False;
4169 except ValueError:
4170 return False;
4171 curloc = catfp.tell();
4172 if(curloc>0):
4173 catfp.seek(0, 0);
4174 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4175 if(curloc>0):
4176 catfp.seek(curloc, 0);
4177 catstring = catheader[0];
4178 catversion = re.findall(r"([\d]+)", catstring);
4179 fprenumfiles = catheader[1];
4180 fnumfiles = int(fprenumfiles, 16);
4181 fprechecksumtype = catheader[2];
4182 fprechecksum = catheader[3];
4183 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4184 fnumfileshex = format(int(fnumfiles), 'x').lower();
4185 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4186 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4187 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4188 fheadtell = len(fileheader);
4189 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4190 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4191 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4192 return False;
4193 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4194 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4195 if(seekto>=fnumfiles):
4196 seekto = fnumfiles - 1;
4197 if(seekto<0):
4198 seekto = 0;
4199 if(seekto>=0):
4200 il = -1;
4201 while(il < seekto):
4202 prefhstart = catfp.tell();
4203 if(formatspecs['new_style']):
4204 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4205 else:
4206 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4207 if(len(preheaderdata)==0):
4208 break;
4209 prefheadsize = int(preheaderdata[0], 16);
4210 prefnumfields = int(preheaderdata[1], 16);
4211 preftype = int(preheaderdata[2], 16);
4212 if(re.findall(r"^[.|/]", preheaderdata[3])):
4213 prefname = preheaderdata[3];
4214 else:
4215 prefname = "./"+preheaderdata[3];
4216 prefbasedir = os.path.dirname(prefname);
4217 preflinkname = preheaderdata[4];
4218 prefsize = int(preheaderdata[5], 16);
4219 prefatime = int(preheaderdata[6], 16);
4220 prefmtime = int(preheaderdata[7], 16);
4221 prefctime = int(preheaderdata[8], 16);
4222 prefbtime = int(preheaderdata[9], 16);
4223 prefmode = int(preheaderdata[10], 16);
4224 prefchmode = stat.S_IMODE(prefmode);
4225 preftypemod = stat.S_IFMT(prefmode);
4226 prefwinattributes = int(preheaderdata[11], 16);
4227 prefcompression = preheaderdata[12];
4228 prefcsize = int(preheaderdata[13], 16);
4229 prefuid = int(preheaderdata[14], 16);
4230 prefuname = preheaderdata[15];
4231 prefgid = int(preheaderdata[16], 16);
4232 prefgname = preheaderdata[17];
4233 fid = int(preheaderdata[18], 16);
4234 finode = int(preheaderdata[19], 16);
4235 flinkcount = int(preheaderdata[20], 16);
4236 prefdev_minor = int(preheaderdata[21], 16);
4237 prefdev_major = int(preheaderdata[22], 16);
4238 prefrdev_minor = int(preheaderdata[23], 16);
4239 prefrdev_major = int(preheaderdata[24], 16);
4240 prefseeknextfile = preheaderdata[25];
4241 prefextrasize = int(preheaderdata[26], 16);
4242 prefextrafields = int(preheaderdata[27], 16);
4243 extrafieldslist = [];
4244 extrastart = 28;
4245 extraend = extrastart + prefextrafields;
4246 extrafieldslist = [];
4247 if(extrastart<extraend):
4248 extrafieldslist.append(preheaderdata[extrastart]);
4249 extrastart = extrastart + 1;
4250 prefcs = preheaderdata[-2].lower();
4251 prenewfcs = preheaderdata[-1].lower();
4252 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4253 if(prefcs!=prenewfcs and not skipchecksum):
4254 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4255 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4256 return False;
4257 valid_archive = False;
4258 invalid_archive = True;
4259 prefhend = catfp.tell() - 1;
4260 prefcontentstart = catfp.tell();
4261 prefcontents = "";
4262 pyhascontents = False;
4263 if(prefsize>0):
4264 if(prefcompression):
4265 prefcontents = catfp.read(prefsize);
4266 else:
4267 prefcontents = catfp.read(prefcsize);
4268 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4269 pyhascontents = True;
4270 if(prefccs!=prenewfccs and not skipchecksum):
4271 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4272 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4273 return False;
4274 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4275 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4276 if(abs(fseeknextasnum)==0):
4277 pass;
4278 catfp.seek(fseeknextasnum, 1);
4279 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4280 fseeknextasnum = int(prefseeknextfile);
4281 if(abs(fseeknextasnum)==0):
4282 pass;
4283 catfp.seek(fseeknextasnum, 1);
4284 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4285 fseeknextasnum = int(prefseeknextfile);
4286 if(abs(fseeknextasnum)==0):
4287 pass;
4288 catfp.seek(fseeknextasnum, 0);
4289 else:
4290 return False;
4291 il = il + 1;
4292 catfp.seek(seekstart, 0);
4293 fileidnum = il;
4294 catfheadsize = int(preheaderdata[0], 16);
4295 catfnumfields = int(preheaderdata[1], 16);
4296 catftype = int(preheaderdata[2], 16);
4297 if(re.findall(r"^[.|/]", preheaderdata[3])):
4298 catfname = preheaderdata[3];
4299 else:
4300 catfname = "./"+preheaderdata[3];
4301 catflinkname = preheaderdata[4];
4302 catfsize = int(preheaderdata[5], 16);
4303 catfbasedir = os.path.dirname(catfname);
4304 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4305 if(returnfp):
4306 catlist.update({'catfp': catfp});
4307 else:
4308 catfp.close();
4309 return catlist;
4310
4311 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4312
4313 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4314 formatspecs = FormatSpecsListToDict(formatspecs);
4315 if(hasattr(infile, "read") or hasattr(infile, "write")):
4316 catfp = infile;
4317 catfp.seek(0, 0);
4318 catfp = UncompressArchiveFile(catfp, formatspecs);
4319 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4320 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4321 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4322 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4323 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4324 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4325 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4326 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4327 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4328 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4329 return False;
4330 if(not catfp):
4331 return False;
4332 catfp.seek(0, 0);
4333 elif(infile=="-"):
4334 catfp = BytesIO();
4335 if(hasattr(sys.stdin, "buffer")):
4336 shutil.copyfileobj(sys.stdin.buffer, catfp);
4337 else:
4338 shutil.copyfileobj(sys.stdin, catfp);
4339 catfp.seek(0, 0);
4340 catfp = UncompressArchiveFile(catfp, formatspecs);
4341 if(not catfp):
4342 return False;
4343 catfp.seek(0, 0);
4344 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4345 catfp = download_file_from_internet_file(infile);
4346 catfp = UncompressArchiveFile(catfp, formatspecs);
4347 catfp.seek(0, 0);
4348 if(not catfp):
4349 return False;
4350 catfp.seek(0, 0);
4351 else:
4352 infile = RemoveWindowsPath(infile);
4353 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4354 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4355 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4356 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4357 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4358 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4359 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4360 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4361 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4362 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4363 return False;
4364 compresscheck = CheckCompressionType(infile, formatspecs, True);
4365 if(not compresscheck):
4366 fextname = os.path.splitext(infile)[1];
4367 if(fextname==".gz"):
4368 compresscheck = "gzip";
4369 elif(fextname==".bz2"):
4370 compresscheck = "bzip2";
4371 elif(fextname==".zst"):
4372 compresscheck = "zstd";
4373 elif(fextname==".lz4" or fextname==".clz4"):
4374 compresscheck = "lz4";
4375 elif(fextname==".lzo" or fextname==".lzop"):
4376 compresscheck = "lzo";
4377 elif(fextname==".lzma" or fextname==".xz"):
4378 compresscheck = "lzma";
4379 else:
4380 return False;
4381 if(not compresscheck):
4382 return False;
4383 catfp = UncompressFile(infile, formatspecs, "rb");
4384 '''
4385 try:
4386 catfp.seek(0, 2);
4387 except OSError:
4388 SeekToEndOfFile(catfp);
4389 except ValueError:
4390 SeekToEndOfFile(catfp);
4391 CatSize = catfp.tell();
4392 CatSizeEnd = CatSize;
4393 '''
4394 try:
4395 catfp.seek(0, 0);
4396 except OSError:
4397 return False;
4398 except ValueError:
4399 return False;
4400 curloc = catfp.tell();
4401 if(curloc>0):
4402 catfp.seek(0, 0);
4403 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4404 if(curloc>0):
4405 catfp.seek(curloc, 0);
4406 catstring = catheader[0];
4407 catversion = re.findall(r"([\d]+)", catstring);
4408 fprenumfiles = catheader[1];
4409 fnumfiles = int(fprenumfiles, 16);
4410 fprechecksumtype = catheader[2];
4411 fprechecksum = catheader[3];
4412 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4413 fnumfileshex = format(int(fnumfiles), 'x').lower();
4414 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4415 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4416 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4417 fheadtell = len(fileheader);
4418 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4419 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4420 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4421 return False;
4422 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4423 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4424 seekto = fnumfiles - 1
4425 filefound = False;
4426 if(seekto>=0):
4427 il = -1;
4428 while(il < seekto):
4429 prefhstart = catfp.tell();
4430 if(formatspecs['new_style']):
4431 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4432 else:
4433 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4434 if(len(preheaderdata)==0):
4435 break;
4436 prefheadsize = int(preheaderdata[0], 16);
4437 prefnumfields = int(preheaderdata[1], 16);
4438 preftype = int(preheaderdata[2], 16);
4439 if(re.findall(r"^[.|/]", preheaderdata[3])):
4440 prefname = preheaderdata[3];
4441 else:
4442 prefname = "./"+preheaderdata[3];
4443 prefbasedir = os.path.dirname(prefname);
4444 preflinkname = preheaderdata[4];
4445 prefsize = int(preheaderdata[5], 16);
4446 prefatime = int(preheaderdata[6], 16);
4447 prefmtime = int(preheaderdata[7], 16);
4448 prefctime = int(preheaderdata[8], 16);
4449 prefbtime = int(preheaderdata[9], 16);
4450 prefmode = int(preheaderdata[10], 16);
4451 prefchmode = stat.S_IMODE(prefmode);
4452 preftypemod = stat.S_IFMT(prefmode);
4453 prefwinattributes = int(preheaderdata[11], 16);
4454 prefcompression = preheaderdata[12];
4455 prefcsize = int(preheaderdata[13], 16);
4456 prefuid = int(preheaderdata[14], 16);
4457 prefuname = preheaderdata[15];
4458 prefgid = int(preheaderdata[16], 16);
4459 prefgname = preheaderdata[17];
4460 fid = int(preheaderdata[18], 16);
4461 finode = int(preheaderdata[19], 16);
4462 flinkcount = int(preheaderdata[20], 16);
4463 prefdev_minor = int(preheaderdata[21], 16);
4464 prefdev_major = int(preheaderdata[22], 16);
4465 prefrdev_minor = int(preheaderdata[23], 16);
4466 prefrdev_major = int(preheaderdata[24], 16);
4467 prefseeknextfile = preheaderdata[25];
4468 prefextrasize = int(preheaderdata[26], 16);
4469 prefextrafields = int(preheaderdata[27], 16);
4470 extrafieldslist = [];
4471 extrastart = 28;
4472 extraend = extrastart + prefextrafields;
4473 extrafieldslist = [];
4474 if(extrastart<extraend):
4475 extrafieldslist.append(preheaderdata[extrastart]);
4476 extrastart = extrastart + 1;
4477 prefcs = preheaderdata[-2].lower();
4478 prenewfcs = preheaderdata[-1].lower();
4479 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4480 if(prefcs!=prenewfcs and not skipchecksum):
4481 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4482 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4483 return False;
4484 valid_archive = False;
4485 invalid_archive = True;
4486 prefhend = catfp.tell() - 1;
4487 prefcontentstart = catfp.tell();
4488 prefcontents = "";
4489 pyhascontents = False;
4490 if(prefsize>0):
4491 if(prefcompression):
4492 prefcontents = catfp.read(prefsize);
4493 else:
4494 prefcontents = catfp.read(prefcsize);
4495 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4496 pyhascontents = True;
4497 if(prefccs!=prenewfccs and not skipchecksum):
4498 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4499 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4500 return False;
4501 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4502 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4503 if(abs(fseeknextasnum)==0):
4504 pass;
4505 catfp.seek(fseeknextasnum, 1);
4506 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4507 fseeknextasnum = int(prefseeknextfile);
4508 if(abs(fseeknextasnum)==0):
4509 pass;
4510 catfp.seek(fseeknextasnum, 1);
4511 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4512 fseeknextasnum = int(prefseeknextfile);
4513 if(abs(fseeknextasnum)==0):
4514 pass;
4515 catfp.seek(fseeknextasnum, 0);
4516 else:
4517 return False;
4518 il = il + 1;
4519 filefound = False;
4520 prefname = preheaderdata[2];
4521 if(re.findall(r"^[.|/]", preheaderdata[2])):
4522 prefname = preheaderdata[2];
4523 else:
4524 prefname = "./"+preheaderdata[2];
4525 if(prefname==seekfile):
4526 filefound = True;
4527 break;
4528 catfp.seek(seekstart, 0);
4529 fileidnum = il;
4530 catfheadsize = int(preheaderdata[0], 16);
4531 catfnumfields = int(preheaderdata[1], 16);
4532 catftype = int(preheaderdata[2], 16);
4533 if(re.findall(r"^[.|/]", preheaderdata[3])):
4534 catfname = preheaderdata[3];
4535 else:
4536 catfname = "./"+preheaderdata[3];
4537 catflinkname = preheaderdata[4];
4538 catfsize = int(preheaderdata[5], 16);
4539 catfbasedir = os.path.dirname(catfname);
4540 if(filefound):
4541 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4542 else:
4543 return False;
4544 if(returnfp):
4545 catlist.update({'catfp': catfp});
4546 else:
4547 catfp.close();
4548 return catlist;
4549
4550 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4551
4552 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4553 formatspecs = FormatSpecsListToDict(formatspecs);
4554 if(verbose):
4555 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4556 if(hasattr(infile, "read") or hasattr(infile, "write")):
4557 catfp = infile;
4558 catfp.seek(0, 0);
4559 catfp = UncompressArchiveFile(catfp, formatspecs);
4560 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4561 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4562 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4563 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4564 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4565 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4566 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4567 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4568 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4569 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4570 return False;
4571 if(not catfp):
4572 return False;
4573 catfp.seek(0, 0);
4574 elif(infile=="-"):
4575 catfp = BytesIO();
4576 if(hasattr(sys.stdin, "buffer")):
4577 shutil.copyfileobj(sys.stdin.buffer, catfp);
4578 else:
4579 shutil.copyfileobj(sys.stdin, catfp);
4580 catfp.seek(0, 0);
4581 catfp = UncompressArchiveFile(catfp, formatspecs);
4582 if(not catfp):
4583 return False;
4584 catfp.seek(0, 0);
4585 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4586 catfp = download_file_from_internet_file(infile);
4587 catfp = UncompressArchiveFile(catfp, formatspecs);
4588 catfp.seek(0, 0);
4589 if(not catfp):
4590 return False;
4591 catfp.seek(0, 0);
4592 else:
4593 infile = RemoveWindowsPath(infile);
4594 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4595 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4596 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4597 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4598 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4599 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4600 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4601 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4602 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4603 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4604 return False;
4605 compresscheck = CheckCompressionType(infile, formatspecs, True);
4606 if(not compresscheck):
4607 fextname = os.path.splitext(infile)[1];
4608 if(fextname==".gz"):
4609 compresscheck = "gzip";
4610 elif(fextname==".bz2"):
4611 compresscheck = "bzip2";
4612 elif(fextname==".zst"):
4613 compresscheck = "zstd";
4614 elif(fextname==".lz4" or fextname==".clz4"):
4615 compresscheck = "lz4";
4616 elif(fextname==".lzo" or fextname==".lzop"):
4617 compresscheck = "lzo";
4618 elif(fextname==".lzma" or fextname==".xz"):
4619 compresscheck = "lzma";
4620 else:
4621 return False;
4622 if(not compresscheck):
4623 return False;
4624 catfp = UncompressFile(infile, formatspecs, "rb");
4625 '''
4626 try:
4627 catfp.seek(0, 2);
4628 except OSError:
4629 SeekToEndOfFile(catfp);
4630 except ValueError:
4631 SeekToEndOfFile(catfp);
4632 CatSize = catfp.tell();
4633 CatSizeEnd = CatSize;
4634 '''
4635 try:
4636 catfp.seek(0, 0);
4637 except OSError:
4638 return False;
4639 except ValueError:
4640 return False;
4641 curloc = catfp.tell();
4642 if(curloc>0):
4643 catfp.seek(0, 0);
4644 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4645 if(curloc>0):
4646 catfp.seek(curloc, 0);
4647 catstring = catheader[0];
4648 catversion = re.findall(r"([\d]+)", catstring);
4649 fprenumfiles = catheader[1];
4650 fnumfiles = int(fprenumfiles, 16);
4651 fprechecksumtype = catheader[2];
4652 fprechecksum = catheader[3];
4653 il = 0;
4654 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4655 fnumfileshex = format(int(fnumfiles), 'x').lower();
4656 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4657 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4658 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4659 valid_archive = True;
4660 invalid_archive = False;
4661 if(verbose):
4662 VerbosePrintOut(infile);
4663 VerbosePrintOut("Number of Records " + str(fnumfiles));
4664 if(fprechecksum==catfileheadercshex):
4665 if(verbose):
4666 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4667 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4668 else:
4669 if(verbose):
4670 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4671 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4672 valid_archive = False;
4673 invalid_archive = True;
4674 if(verbose):
4675 VerbosePrintOut("");
4676 while(il<fnumfiles):
4677 catfhstart = catfp.tell();
4678 if(formatspecs['new_style']):
4679 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4680 else:
4681 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4682 if(len(catheaderdata)==0):
4683 break;
4684 catfheadsize = int(catheaderdata[0], 16);
4685 catfnumfields = int(catheaderdata[1], 16);
4686 catftype = int(catheaderdata[2], 16);
4687 if(re.findall(r"^[.|/]", catheaderdata[3])):
4688 catfname = catheaderdata[3];
4689 else:
4690 catfname = "./"+catheaderdata[3];
4691 catfbasedir = os.path.dirname(catfname);
4692 catflinkname = catheaderdata[4];
4693 catfsize = int(catheaderdata[5], 16);
4694 catfatime = int(catheaderdata[6], 16);
4695 catfmtime = int(catheaderdata[7], 16);
4696 catfctime = int(catheaderdata[8], 16);
4697 catfbtime = int(catheaderdata[9], 16);
4698 catfmode = int(catheaderdata[10], 16);
4699 catfchmode = stat.S_IMODE(catfmode);
4700 catftypemod = stat.S_IFMT(catfmode);
4701 catfwinattributes = int(catheaderdata[11], 16);
4702 catfcompression = catheaderdata[12];
4703 catfcsize = int(catheaderdata[13], 16);
4704 catfuid = int(catheaderdata[14], 16);
4705 catfuname = catheaderdata[15];
4706 catfgid = int(catheaderdata[16], 16);
4707 catfgname = catheaderdata[17];
4708 fid = int(catheaderdata[18], 16);
4709 finode = int(catheaderdata[19], 16);
4710 flinkcount = int(catheaderdata[20], 16);
4711 catfdev_minor = int(catheaderdata[21], 16);
4712 catfdev_major = int(catheaderdata[22], 16);
4713 catfrdev_minor = int(catheaderdata[23], 16);
4714 catfrdev_major = int(catheaderdata[24], 16);
4715 catfseeknextfile = catheaderdata[25];
4716 catfextrasize = int(catheaderdata[26], 16);
4717 catfextrafields = int(catheaderdata[27], 16);
4718 extrafieldslist = [];
4719 extrastart = 28;
4720 extraend = extrastart + catfextrafields;
4721 extrafieldslist = [];
4722 if(extrastart<extraend):
4723 extrafieldslist.append(catheaderdata[extrastart]);
4724 extrastart = extrastart + 1;
4725 catfcs = catheaderdata[-2].lower();
4726 catfccs = catheaderdata[-1].lower();
4727 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4728 if(verbose):
4729 VerbosePrintOut(catfname);
4730 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4731 if(catfcs==catnewfcs):
4732 if(verbose):
4733 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4734 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4735 else:
4736 if(verbose):
4737 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4738 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4739 valid_archive = False;
4740 invalid_archive = True;
4741 catfhend = catfp.tell() - 1;
4742 catfcontentstart = catfp.tell();
4743 catfcontents = "";
4744 pyhascontents = False;
4745 if(catfsize>0):
4746 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4747 catfcontents = catfp.read(catfsize);
4748 else:
4749 catfcontents = catfp.read(catfcsize);
4750 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4751 pyhascontents = True;
4752 if(catfccs==catnewfccs):
4753 if(verbose):
4754 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4755 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4756 else:
4757 if(verbose):
4758 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4759 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4760 valid_archive = False;
4761 invalid_archive = True;
4762 if(verbose):
4763 VerbosePrintOut("");
4764 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
4765 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
4766 if(abs(fseeknextasnum)==0):
4767 pass;
4768 catfp.seek(fseeknextasnum, 1);
4769 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
4770 fseeknextasnum = int(catfseeknextfile);
4771 if(abs(fseeknextasnum)==0):
4772 pass;
4773 catfp.seek(fseeknextasnum, 1);
4774 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
4775 fseeknextasnum = int(catfseeknextfile);
4776 if(abs(fseeknextasnum)==0):
4777 pass;
4778 catfp.seek(fseeknextasnum, 0);
4779 else:
4780 return False;
4781 il = il + 1;
4782 if(valid_archive):
4783 if(returnfp):
4784 return catfp;
4785 else:
4786 catfp.close();
4787 return True;
4788 else:
4789 catfp.close();
4790 return False;
4791
4792 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4793
4794 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4795 formatspecs = FormatSpecsListToDict(formatspecs);
4796 if(hasattr(infile, "read") or hasattr(infile, "write")):
4797 catfp = infile;
4798 catfp.seek(0, 0);
4799 catfp = UncompressArchiveFile(catfp, formatspecs);
4800 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4801 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4802 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4803 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4804 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4805 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4806 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4807 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4808 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4809 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4810 return False;
4811 if(not catfp):
4812 return False;
4813 catfp.seek(0, 0);
4814 elif(infile=="-"):
4815 catfp = BytesIO();
4816 if(hasattr(sys.stdin, "buffer")):
4817 shutil.copyfileobj(sys.stdin.buffer, catfp);
4818 else:
4819 shutil.copyfileobj(sys.stdin, catfp);
4820 catfp.seek(0, 0);
4821 catfp = UncompressArchiveFile(catfp, formatspecs);
4822 if(not catfp):
4823 return False;
4824 catfp.seek(0, 0);
4825 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4826 catfp = download_file_from_internet_file(infile);
4827 catfp = UncompressArchiveFile(catfp, formatspecs);
4828 catfp.seek(0, 0);
4829 if(not catfp):
4830 return False;
4831 catfp.seek(0, 0);
4832 else:
4833 infile = RemoveWindowsPath(infile);
4834 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4835 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4836 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4837 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4838 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4839 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4840 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4841 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4842 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4843 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4844 return False;
4845 compresscheck = CheckCompressionType(infile, formatspecs, True);
4846 if(not compresscheck):
4847 fextname = os.path.splitext(infile)[1];
4848 if(fextname==".gz"):
4849 compresscheck = "gzip";
4850 elif(fextname==".bz2"):
4851 compresscheck = "bzip2";
4852 elif(fextname==".zst"):
4853 compresscheck = "zstd";
4854 elif(fextname==".lz4" or fextname==".clz4"):
4855 compresscheck = "lz4";
4856 elif(fextname==".lzo" or fextname==".lzop"):
4857 compresscheck = "lzo";
4858 elif(fextname==".lzma" or fextname==".xz"):
4859 compresscheck = "lzma";
4860 else:
4861 return False;
4862 if(not compresscheck):
4863 return False;
4864 catfp = UncompressFile(infile, formatspecs, "rb");
4865 '''
4866 try:
4867 catfp.seek(0, 2);
4868 except OSError:
4869 SeekToEndOfFile(catfp);
4870 except ValueError:
4871 SeekToEndOfFile(catfp);
4872 CatSize = catfp.tell();
4873 CatSizeEnd = CatSize;
4874 '''
4875 try:
4876 catfp.seek(0, 0);
4877 except OSError:
4878 return False;
4879 except ValueError:
4880 return False;
4881 curloc = catfp.tell();
4882 if(curloc>0):
4883 catfp.seek(0, 0);
4884 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4885 if(curloc>0):
4886 catfp.seek(curloc, 0);
4887 catstring = catheader[0];
4888 catversion = re.findall(r"([\d]+)", catstring);
4889 fprenumfiles = catheader[1];
4890 fnumfiles = int(fprenumfiles, 16);
4891 fprechecksumtype = catheader[2];
4892 fprechecksum = catheader[3];
4893 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4894 fnumfileshex = format(int(fnumfiles), 'x').lower();
4895 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4896 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4897 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4898 fheadtell = len(fileheader);
4899 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4900 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4901 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4902 return False;
4903 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4904 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
4905 if(seekstart<0 and seekstart>fnumfiles):
4906 seekstart = 0;
4907 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4908 seekend = fnumfiles;
4909 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4910 seekend = fnumfiles - abs(seekend);
4911 if(seekstart>0):
4912 il = 0;
4913 while(il < seekstart):
4914 prefhstart = catfp.tell();
4915 if(formatspecs['new_style']):
4916 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4917 else:
4918 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4919 if(len(preheaderdata)==0):
4920 break;
4921 prefheadsize = int(preheaderdata[0], 16);
4922 prefnumfields = int(preheaderdata[1], 16);
4923 if(re.findall(r"^[.|/]", preheaderdata[3])):
4924 prefname = preheaderdata[3];
4925 else:
4926 prefname = "./"+preheaderdata[3];
4927 prefsize = int(preheaderdata[5], 16);
4928 prefcompression = preheaderdata[12];
4929 prefcsize = int(preheaderdata[13], 16);
4930 prefseeknextfile = preheaderdata[25];
4931 prefextrasize = int(preheaderdata[26], 16);
4932 prefextrafields = int(preheaderdata[27], 16);
4933 extrafieldslist = [];
4934 extrastart = 28;
4935 extraend = extrastart + prefextrafields;
4936 extrafieldslist = [];
4937 if(extrastart<extraend):
4938 extrafieldslist.append(preheaderdata[extrastart]);
4939 extrastart = extrastart + 1;
4940 prefcs = preheaderdata[-2].lower();
4941 prenewfcs = preheaderdata[-1].lower();
4942 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4943 if(prefcs!=prenewfcs and not skipchecksum):
4944 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4945 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4946 return False;
4947 valid_archive = False;
4948 invalid_archive = True;
4949 prefhend = catfp.tell() - 1;
4950 prefcontentstart = catfp.tell();
4951 prefcontents = "";
4952 pyhascontents = False;
4953 if(prefsize>0):
4954 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4955 prefcontents = catfp.read(prefsize);
4956 else:
4957 prefcontents = catfp.read(prefcsize);
4958 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4959 pyhascontents = True;
4960 if(prefccs!=prenewfccs and not skipchecksum):
4961 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4962 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4963 return False;
4964 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4965 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4966 if(abs(fseeknextasnum)==0):
4967 pass;
4968 catfp.seek(fseeknextasnum, 1);
4969 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4970 fseeknextasnum = int(prefseeknextfile);
4971 if(abs(fseeknextasnum)==0):
4972 pass;
4973 catfp.seek(fseeknextasnum, 1);
4974 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4975 fseeknextasnum = int(prefseeknextfile);
4976 if(abs(fseeknextasnum)==0):
4977 pass;
4978 catfp.seek(fseeknextasnum, 0);
4979 else:
4980 return False;
4981 il = il + 1;
4982 fileidnum = seekstart;
4983 realidnum = 0;
4984 while(fileidnum<seekend):
4985 catfhstart = catfp.tell();
4986 if(formatspecs['new_style']):
4987 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4988 else:
4989 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4990 if(len(catheaderdata)==0):
4991 break;
4992 catfheadsize = int(catheaderdata[0], 16);
4993 catfnumfields = int(catheaderdata[1], 16);
4994 catftype = int(catheaderdata[2], 16);
4995 if(re.findall(r"^[.|/]", catheaderdata[3])):
4996 catfname = catheaderdata[3];
4997 else:
4998 catfname = "./"+catheaderdata[3];
4999 catfbasedir = os.path.dirname(catfname);
5000 catflinkname = catheaderdata[4];
5001 catfsize = int(catheaderdata[5], 16);
5002 catfatime = int(catheaderdata[6], 16);
5003 catfmtime = int(catheaderdata[7], 16);
5004 catfctime = int(catheaderdata[8], 16);
5005 catfbtime = int(catheaderdata[9], 16);
5006 catfmode = int(catheaderdata[10], 16);
5007 catfchmode = stat.S_IMODE(catfmode);
5008 catftypemod = stat.S_IFMT(catfmode);
5009 catfwinattributes = int(catheaderdata[11], 16);
5010 catfcompression = catheaderdata[12];
5011 catfcsize = int(catheaderdata[13], 16);
5012 catfuid = int(catheaderdata[14], 16);
5013 catfuname = catheaderdata[15];
5014 catfgid = int(catheaderdata[16], 16);
5015 catfgname = catheaderdata[17];
5016 catfid = int(catheaderdata[18], 16);
5017 catfinode = int(catheaderdata[19], 16);
5018 catflinkcount = int(catheaderdata[20], 16);
5019 catfdev_minor = int(catheaderdata[21], 16);
5020 catfdev_major = int(catheaderdata[22], 16);
5021 catfrdev_minor = int(catheaderdata[23], 16);
5022 catfrdev_major = int(catheaderdata[24], 16);
5023 catfseeknextfile = catheaderdata[25];
5024 catfextrasize = int(catheaderdata[26], 16);
5025 catfextrafields = int(catheaderdata[27], 16);
5026 extrafieldslist = [];
5027 extrastart = 28;
5028 extraend = extrastart + catfextrafields;
5029 extrafieldslist = [];
5030 if(extrastart<extraend):
5031 extrafieldslist.append(catheaderdata[extrastart]);
5032 extrastart = extrastart + 1;
5033 catfcs = catheaderdata[-2].lower();
5034 catfccs = catheaderdata[-1].lower();
5035 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5036 if(catfcs!=catnewfcs and not skipchecksum):
5037 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
5038 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5039 return False;
5040 catfhend = catfp.tell() - 1;
5041 catfcontentstart = catfp.tell();
5042 catfcontents = BytesIO();
5043 pyhascontents = False;
5044 if(catfsize>0 and not listonly):
5045 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5046 catfcontents.write(catfp.read(catfsize));
5047 else:
5048 catfcontents.write(catfp.read(catfcsize));
5049 catfcontents.seek(0, 0);
5050 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5051 pyhascontents = True;
5052 if(catfccs!=catnewfccs and skipchecksum):
5053 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5054 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5055 return False;
5056 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5057 pass;
5058 else:
5059 catfcontents.seek(0, 0);
5060 if(uncompress):
5061 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5062 catfcontents.seek(0, 0);
5063 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5064 if(catfsize>0 and listonly):
5065 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5066 catfp.seek(catfsize, 1);
5067 else:
5068 catfp.seek(catfcsize, 1);
5069 pyhascontents = False;
5070 catfcontentend = catfp.tell();
5071 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5072 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5073 if(abs(fseeknextasnum)==0):
5074 pass;
5075 catfp.seek(fseeknextasnum, 1);
5076 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5077 fseeknextasnum = int(catfseeknextfile);
5078 if(abs(fseeknextasnum)==0):
5079 pass;
5080 catfp.seek(fseeknextasnum, 1);
5081 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5082 fseeknextasnum = int(catfseeknextfile);
5083 if(abs(fseeknextasnum)==0):
5084 pass;
5085 catfp.seek(fseeknextasnum, 0);
5086 else:
5087 return False;
5088 catfcontents.seek(0, 0);
5089 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
5090 fileidnum = fileidnum + 1;
5091 realidnum = realidnum + 1;
5092 if(returnfp):
5093 catlist.update({'catfp': catfp});
5094 else:
5095 catfp.close();
5096 return catlist;
5097
5098 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5099
5100 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5101 formatspecs = FormatSpecsListToDict(formatspecs);
5102 catfp = BytesIO(catstr);
5103 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5104 return listcatfiles;
5105
5106 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5107
5108 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5109 formatspecs = FormatSpecsListToDict(formatspecs);
5110 catfp = BytesIO();
5111 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5112 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5113 return listcatfiles;
5114
5115 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5116 formatspecs = FormatSpecsListToDict(formatspecs);
5117 catfp = BytesIO();
5118 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5119 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5120 return listcatfiles;
5121
5122 if(not rarfile_support):
5123 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5124 return False;
5125
5126 if(rarfile_support):
5127 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5128 formatspecs = FormatSpecsListToDict(formatspecs);
5129 catfp = BytesIO();
5130 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5131 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5132 return listcatfiles;
5133
5134 if(not py7zr_support):
5135 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5136 return False;
5137
5138 if(py7zr_support):
5139 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5140 formatspecs = FormatSpecsListToDict(formatspecs);
5141 catfp = BytesIO();
5142 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5143 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5144 return listcatfiles;
5145
5146 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5147 formatspecs = FormatSpecsListToDict(formatspecs);
5148 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5149 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5150 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5151 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5152 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5153 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5154 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5155 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5156 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5157 elif(checkcompressfile=="catfile"):
5158 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5159 else:
5160 return False;
5161 return False;
5162
5163 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5164 formatspecs = FormatSpecsListToDict(formatspecs);
5165 catver = formatspecs['format_ver'];
5166 fileheaderver = str(int(catver.replace(".", "")));
5167 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5168 advancedlist = formatspecs['use_advanced_list'];
5169 altinode = formatspecs['use_alt_inode'];
5170 infilelist = [];
5171 if(infiles=="-"):
5172 for line in sys.stdin:
5173 infilelist.append(line.strip());
5174 infilelist = list(filter(None, infilelist));
5175 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5176 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5177 return False;
5178 with UncompressFile(infiles, formatspecs, "r") as finfile:
5179 for line in finfile:
5180 infilelist.append(line.strip());
5181 infilelist = list(filter(None, infilelist));
5182 else:
5183 if(isinstance(infiles, (list, tuple, ))):
5184 infilelist = list(filter(None, infiles));
5185 elif(isinstance(infiles, (str, ))):
5186 infilelist = list(filter(None, [infiles]));
5187 if(advancedlist):
5188 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5189 else:
5190 GetDirList = ListDir(infilelist, followlink, False);
5191 if(not GetDirList):
5192 return False;
5193 curinode = 0;
5194 curfid = 0;
5195 inodelist = [];
5196 inodetofile = {};
5197 filetoinode = {};
5198 inodetocatinode = {};
5199 fileidnum = 0;
5200 fnumfiles = int(len(GetDirList));
5201 catver = formatspecs['format_ver'];
5202 fileheaderver = str(int(catver.replace(".", "")));
5203 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5204 fnumfileshex = format(int(fnumfiles), 'x').lower();
5205 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5206 catversion = re.findall(r"([\d]+)", fileheader);
5207 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5208 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5209 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5210 fheadtell = len(fileheader);
5211 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5212 for curfname in GetDirList:
5213 catfhstart = fheadtell;
5214 if(re.findall(r"^[.|/]", curfname)):
5215 fname = curfname;
5216 else:
5217 fname = "./"+curfname;
5218 if(verbose):
5219 VerbosePrintOut(fname);
5220 if(not followlink or followlink is None):
5221 fstatinfo = os.lstat(fname);
5222 else:
5223 fstatinfo = os.stat(fname);
5224 fpremode = fstatinfo.st_mode;
5225 finode = fstatinfo.st_ino;
5226 flinkcount = fstatinfo.st_nlink;
5227 ftype = 0;
5228 if(stat.S_ISREG(fpremode)):
5229 ftype = 0;
5230 elif(stat.S_ISLNK(fpremode)):
5231 ftype = 2;
5232 elif(stat.S_ISCHR(fpremode)):
5233 ftype = 3;
5234 elif(stat.S_ISBLK(fpremode)):
5235 ftype = 4;
5236 elif(stat.S_ISDIR(fpremode)):
5237 ftype = 5;
5238 elif(stat.S_ISFIFO(fpremode)):
5239 ftype = 6;
5240 elif(stat.S_ISSOCK(fpremode)):
5241 ftype = 8;
5242 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5243 ftype = 9;
5244 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5245 ftype = 10;
5246 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5247 ftype = 11;
5248 else:
5249 ftype = 0;
5250 flinkname = "";
5251 fbasedir = os.path.dirname(fname);
5252 fcurfid = curfid;
5253 if(not followlink and finode!=0):
5254 if(ftype!=1):
5255 if(finode in inodelist):
5256 ftype = 1;
5257 flinkname = inodetofile[finode];
5258 if(altinode):
5259 fcurinode = finode;
5260 else:
5261 fcurinode = inodetocatinode[finode];
5262 if(finode not in inodelist):
5263 inodelist.append(finode);
5264 inodetofile.update({finode: fname});
5265 inodetocatinode.update({finode: curinode});
5266 if(altinode):
5267 fcurinode = finode;
5268 else:
5269 fcurinode = curinode;
5270 curinode = curinode + 1;
5271 else:
5272 fcurinode = curinode;
5273 curinode = curinode + 1;
5274 curfid = curfid + 1;
5275 if(ftype==2):
5276 flinkname = os.readlink(fname);
5277 fdev = fstatinfo.st_dev;
5278 getfdev = GetDevMajorMinor(fdev);
5279 fdev_minor = getfdev[0];
5280 fdev_major = getfdev[1];
5281 frdev = fstatinfo.st_dev;
5282 if(hasattr(fstatinfo, "st_rdev")):
5283 frdev = fstatinfo.st_rdev;
5284 else:
5285 frdev = fstatinfo.st_dev;
5286 getfrdev = GetDevMajorMinor(frdev);
5287 frdev_minor = getfrdev[0];
5288 frdev_major = getfrdev[1];
5289 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5290 fsize = "0";
5291 if(ftype==0 or ftype==7):
5292 fsize = fstatinfo.st_size;
5293 fatime = fstatinfo.st_atime;
5294 fmtime = fstatinfo.st_mtime;
5295 fctime = fstatinfo.st_ctime;
5296 if(hasattr(fstatinfo, "st_birthtime")):
5297 fbtime = fstatinfo.st_birthtime;
5298 else:
5299 fbtime = fstatinfo.st_ctime;
5300 fmode = fstatinfo.st_mode;
5301 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5302 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5303 fuid = fstatinfo.st_uid;
5304 fgid = fstatinfo.st_gid;
5305 funame = "";
5306 try:
5307 import pwd;
5308 try:
5309 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5310 funame = userinfo.pw_name;
5311 except KeyError:
5312 funame = "";
5313 except ImportError:
5314 funame = "";
5315 fgname = "";
5316 try:
5317 import grp;
5318 try:
5319 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5320 fgname = groupinfo.gr_name;
5321 except KeyError:
5322 fgname = "";
5323 except ImportError:
5324 fgname = "";
5325 fdev_minor = fdev_minor;
5326 fdev_major = fdev_major;
5327 frdev_minor = frdev_minor;
5328 frdev_major = frdev_major;
5329 flinkcount = flinkcount;
5330 if(hasattr(fstatinfo, "st_file_attributes")):
5331 fwinattributes = fstatinfo.st_file_attributes;
5332 else:
5333 fwinattributes = 0;
5334 fcompression = "";
5335 fcsize = 0;
5336 fcontents = BytesIO();
5337 if(ftype==0 or ftype==7):
5338 with open(fname, "rb") as fpc:
5339 shutil.copyfileobj(fpc, fcontents);
5340 if(followlink and (ftype==1 or ftype==2)):
5341 flstatinfo = os.stat(flinkname);
5342 with open(flinkname, "rb") as fpc:
5343 shutil.copyfileobj(fpc, fcontents);
5344 fcontents.seek(0, 0);
5345 ftypehex = format(ftype, 'x').lower();
5346 extrafields = len(extradata);
5347 extrafieldslist = extradata;
5348 catfextrafields = extrafields;
5349 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5350 if(len(extradata)>0):
5351 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5352 extrasizelen = len(extrasizestr);
5353 extrasizelenhex = format(extrasizelen, 'x').lower();
5354 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5355 catoutlen = len(catoutlist) + len(extradata) + 3;
5356 catoutlenhex = format(catoutlen, 'x').lower();
5357 catoutlist.insert(0, catoutlenhex);
5358 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5359 catheaderdata = catoutlist;
5360 if(len(extradata)>0):
5361 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5362 if(fsize==0):
5363 checksumlist = [checksumtype, "none"];
5364 else:
5365 checksumlist = [checksumtype, checksumtype];
5366 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5367 catfnumfields = catoutlen;
5368 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5369 fcontents.seek(0, 0);
5370 if(fsize==0):
5371 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5372 else:
5373 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5374 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5375 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5376 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5377 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5378 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5379 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5380 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5381 catfcontentstart = fheadtell;
5382 fheadtell += len(catfileoutstr) + 1;
5383 catfcontentend = fheadtell - 1;
5384 catfhend = catfcontentend;
5385 fcontents.seek(0, 0);
5386 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5387 pyhascontents = False;
5388 if(int(fsize)>0 and not listonly):
5389 pyhascontents = True;
5390 if(int(fsize)>0 and listonly):
5391 fcontents = BytesIO();
5392 pyhascontents = False;
5393 fcontents.seek(0, 0);
5394 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5395 fileidnum = fileidnum + 1;
5396 return catlist;
5397
5398 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5399 formatspecs = FormatSpecsListToDict(formatspecs);
5400 curinode = 0;
5401 curfid = 0;
5402 inodelist = [];
5403 inodetofile = {};
5404 filetoinode = {};
5405 inodetocatinode = {};
5406 fileidnum = 0;
5407 if(infile=="-"):
5408 infile = BytesIO();
5409 if(hasattr(sys.stdin, "buffer")):
5410 shutil.copyfileobj(sys.stdin.buffer, infile);
5411 else:
5412 shutil.copyfileobj(sys.stdin, infile);
5413 infile.seek(0, 0);
5414 if(not infile):
5415 return False;
5416 infile.seek(0, 0);
5417 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5418 infile = download_file_from_internet_file(infile);
5419 infile.seek(0, 0);
5420 if(not infile):
5421 return False;
5422 infile.seek(0, 0);
5423 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5424 return False;
5425 elif(os.path.exists(infile) and os.path.isfile(infile)):
5426 try:
5427 if(not tarfile.TarFileCheck(infile)):
5428 return False;
5429 except AttributeError:
5430 if(not TarFileCheck(infile)):
5431 return False;
5432 else:
5433 return False;
5434 try:
5435 if(hasattr(infile, "read") or hasattr(infile, "write")):
5436 tarfp = tarfile.open(fileobj=infile, mode="r");
5437 else:
5438 tarfp = tarfile.open(infile, "r");
5439 except FileNotFoundError:
5440 return False;
5441 fnumfiles = int(len(tarfp.getmembers()));
5442 catver = formatspecs['format_ver'];
5443 fileheaderver = str(int(catver.replace(".", "")));
5444 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5445 fnumfileshex = format(int(fnumfiles), 'x').lower();
5446 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5447 catversion = re.findall(r"([\d]+)", fileheader);
5448 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5449 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5450 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5451 fheadtell = len(fileheader);
5452 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5453 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5454 catfhstart = fheadtell;
5455 if(re.findall(r"^[.|/]", member.name)):
5456 fname = member.name;
5457 else:
5458 fname = "./"+member.name;
5459 if(verbose):
5460 VerbosePrintOut(fname);
5461 fpremode = member.mode;
5462 ffullmode = member.mode;
5463 flinkcount = 0;
5464 ftype = 0;
5465 if(member.isreg()):
5466 ffullmode = member.mode + stat.S_IFREG;
5467 ftype = 0;
5468 elif(member.isdev()):
5469 ffullmode = member.mode;
5470 ftype = 7;
5471 elif(member.islnk()):
5472 ffullmode = member.mode + stat.S_IFREG;
5473 ftype = 1;
5474 elif(member.issym()):
5475 ffullmode = member.mode + stat.S_IFLNK;
5476 ftype = 2;
5477 elif(member.ischr()):
5478 ffullmode = member.mode + stat.S_IFCHR;
5479 ftype = 3;
5480 elif(member.isblk()):
5481 ffullmode = member.mode + stat.S_IFBLK;
5482 ftype = 4;
5483 elif(member.isdir()):
5484 ffullmode = member.mode + stat.S_IFDIR;
5485 ftype = 5;
5486 elif(member.isfifo()):
5487 ffullmode = member.mode + stat.S_IFIFO;
5488 ftype = 6;
5489 elif(member.issparse()):
5490 ffullmode = member.mode;
5491 ftype = 12;
5492 else:
5493 ffullmode = member.mode;
5494 ftype = 0;
5495 flinkname = "";
5496 fbasedir = os.path.dirname(fname);
5497 fcurfid = curfid;
5498 fcurinode = curfid;
5499 finode = fcurinode;
5500 curfid = curfid + 1;
5501 if(ftype==2):
5502 flinkname = member.linkname;
5503 fdev_minor = member.devminor;
5504 fdev_major = member.devmajor;
5505 frdev_minor = member.devminor;
5506 frdev_major = member.devmajor;
5507 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5508 fsize = "0";
5509 elif(ftype==0 or ftype==7):
5510 fsize = member.size;
5511 else:
5512 fsize = member.size;
5513 fatime = member.mtime;
5514 fmtime = member.mtime;
5515 fctime = member.mtime;
5516 fbtime = member.mtime;
5517 fmode = ffullmode;
5518 fchmode = stat.S_IMODE(ffullmode);
5519 ftypemod = stat.S_IFMT(ffullmode);
5520 fuid = member.uid;
5521 fgid = member.gid;
5522 funame = member.uname;
5523 fgname = member.gname;
5524 flinkcount = flinkcount;
5525 fwinattributes = int(0);
5526 fcompression = "";
5527 fcsize = 0;
5528 fcontents = BytesIO();
5529 if(ftype==0 or ftype==7):
5530 with tarfp.extractfile(member) as fpc:
5531 shutil.copyfileobj(fpc, fcontents);
5532 fcontents.seek(0, 0);
5533 ftypehex = format(ftype, 'x').lower();
5534 extrafields = len(extradata);
5535 extrafieldslist = extradata;
5536 catfextrafields = extrafields;
5537 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5538 if(len(extradata)>0):
5539 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5540 extrasizelen = len(extrasizestr);
5541 extrasizelenhex = format(extrasizelen, 'x').lower();
5542 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5543 catoutlen = len(catoutlist) + len(extradata) + 3;
5544 catoutlenhex = format(catoutlen, 'x').lower();
5545 catoutlist.insert(0, catoutlenhex);
5546 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5547 catheaderdata = catoutlist;
5548 if(len(extradata)>0):
5549 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5550 if(fsize==0):
5551 checksumlist = [checksumtype, "none"];
5552 else:
5553 checksumlist = [checksumtype, checksumtype];
5554 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5555 catfnumfields = catoutlen;
5556 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5557 fcontents.seek(0, 0);
5558 if(fsize==0):
5559 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5560 else:
5561 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5562 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5563 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5564 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5565 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5566 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5567 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5568 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5569 catfcontentstart = fheadtell;
5570 fheadtell += len(catfileoutstr) + 1;
5571 catfcontentend = fheadtell - 1;
5572 catfhend = catfcontentend;
5573 fcontents.seek(0, 0);
5574 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5575 pyhascontents = False;
5576 if(int(fsize)>0 and not listonly):
5577 pyhascontents = True;
5578 if(int(fsize)>0 and listonly):
5579 fcontents = BytesIO();
5580 pyhascontents = False;
5581 fcontents.seek(0, 0);
5582 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5583 fileidnum = fileidnum + 1;
5584 return catlist;
5585
5586 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5587 formatspecs = FormatSpecsListToDict(formatspecs);
5588 curinode = 0;
5589 curfid = 0;
5590 inodelist = [];
5591 inodetofile = {};
5592 filetoinode = {};
5593 inodetocatinode = {};
5594 fileidnum = 0;
5595 if(infile=="-"):
5596 infile = BytesIO();
5597 if(hasattr(sys.stdin, "buffer")):
5598 shutil.copyfileobj(sys.stdin.buffer, infile);
5599 else:
5600 shutil.copyfileobj(sys.stdin, infile);
5601 infile.seek(0, 0);
5602 if(not infile):
5603 return False;
5604 infile.seek(0, 0);
5605 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5606 infile = download_file_from_internet_file(infile);
5607 infile.seek(0, 0);
5608 if(not infile):
5609 return False;
5610 infile.seek(0, 0);
5611 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5612 return False;
5613 else:
5614 return False;
5615 if(not zipfile.is_zipfile(infile)):
5616 return False;
5617 try:
5618 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5619 except FileNotFoundError:
5620 return False;
5621 ziptest = zipfp.testzip();
5622 if(ziptest):
5623 VerbosePrintOut("Bad file found!");
5624 fnumfiles = int(len(zipfp.infolist()));
5625 catver = formatspecs['format_ver'];
5626 fileheaderver = str(int(catver.replace(".", "")));
5627 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5628 catversion = re.findall(r"([\d]+)", fileheader);
5629 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5630 fnumfileshex = format(int(fnumfiles), 'x').lower();
5631 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5632 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5633 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5634 fheadtell = len(fileheader);
5635 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5636 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5637 catfhstart = fheadtell;
5638 if(re.findall(r"^[.|/]", member.filename)):
5639 fname = member.filename;
5640 else:
5641 fname = "./"+member.filename;
5642 zipinfo = zipfp.getinfo(member.filename);
5643 if(verbose):
5644 VerbosePrintOut(fname);
5645 if(not member.is_dir()):
5646 fpremode = stat.S_IFREG + 438;
5647 elif(member.is_dir()):
5648 fpremode = stat.S_IFDIR + 511;
5649 flinkcount = 0;
5650 ftype = 0;
5651 if(not member.is_dir()):
5652 ftype = 0;
5653 elif(member.is_dir()):
5654 ftype = 5;
5655 flinkname = "";
5656 fbasedir = os.path.dirname(fname);
5657 fcurfid = curfid;
5658 fcurinode = curfid;
5659 finode = fcurinode;
5660 curfid = curfid + 1;
5661 fdev_minor = 0;
5662 fdev_major = 0;
5663 frdev_minor = 0;
5664 frdev_major = 0;
5665 if(ftype==5):
5666 fsize = "0";
5667 elif(ftype==0):
5668 fsize = member.file_size;
5669 else:
5670 fsize = member.file_size;
5671 fatime = time.mktime(member.date_time + (0, 0, -1));
5672 fmtime = time.mktime(member.date_time + (0, 0, -1));
5673 fctime = time.mktime(member.date_time + (0, 0, -1));
5674 fbtime = time.mktime(member.date_time + (0, 0, -1));
5675 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5676 fwinattributes = int(zipinfo.external_attr);
5677 if(not member.is_dir()):
5678 fmode = int(stat.S_IFREG + 438);
5679 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5680 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5681 elif(member.is_dir()):
5682 fmode = int(stat.S_IFDIR + 511);
5683 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5684 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5685 elif(zipinfo.create_system==3):
5686 fwinattributes = int(0);
5687 fmode = int(zipinfo.external_attr);
5688 else:
5689 fwinattributes = int(0);
5690 if(not member.is_dir()):
5691 fmode = int(stat.S_IFREG + 438);
5692 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5693 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5694 elif(member.is_dir()):
5695 fmode = int(stat.S_IFDIR + 511);
5696 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5697 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5698 fcompression = "";
5699 fcsize = 0;
5700 try:
5701 fuid = os.getuid();
5702 except AttributeError:
5703 fuid = 0;
5704 except KeyError:
5705 fuid = 0;
5706 try:
5707 fgid = os.getgid();
5708 except AttributeError:
5709 fgid = 0;
5710 except KeyError:
5711 fgid = 0;
5712 try:
5713 import pwd;
5714 try:
5715 userinfo = pwd.getpwuid(os.getuid());
5716 funame = userinfo.pw_name;
5717 except KeyError:
5718 funame = "";
5719 except AttributeError:
5720 funame = "";
5721 except ImportError:
5722 funame = "";
5723 fgname = "";
5724 try:
5725 import grp;
5726 try:
5727 groupinfo = grp.getgrgid(os.getgid());
5728 fgname = groupinfo.gr_name;
5729 except KeyError:
5730 fgname = "";
5731 except AttributeError:
5732 fgname = "";
5733 except ImportError:
5734 fgname = "";
5735 fcontents = BytesIO();
5736 if(ftype==0):
5737 fcontents.write(zipfp.read(member.filename));
5738 fcontents.seek(0, 0);
5739 ftypehex = format(ftype, 'x').lower();
5740 extrafields = len(extradata);
5741 extrafieldslist = extradata;
5742 catfextrafields = extrafields;
5743 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5744 if(len(extradata)>0):
5745 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5746 extrasizelen = len(extrasizestr);
5747 extrasizelenhex = format(extrasizelen, 'x').lower();
5748 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5749 catoutlen = len(catoutlist) + len(extradata) + 3;
5750 catoutlenhex = format(catoutlen, 'x').lower();
5751 catoutlist.insert(0, catoutlenhex);
5752 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5753 catheaderdata = catoutlist;
5754 if(len(extradata)>0):
5755 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5756 if(fsize==0):
5757 checksumlist = [checksumtype, "none"];
5758 else:
5759 checksumlist = [checksumtype, checksumtype];
5760 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5761 catfnumfields = catoutlen;
5762 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5763 fcontents.seek(0, 0);
5764 if(fsize==0):
5765 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5766 else:
5767 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5768 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5769 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5770 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5771 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5772 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5773 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5774 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5775 catfcontentstart = fheadtell;
5776 fheadtell += len(catfileoutstr) + 1;
5777 catfcontentend = fheadtell - 1;
5778 catfhend = catfcontentend;
5779 fcontents.seek(0, 0);
5780 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5781 pyhascontents = False;
5782 if(int(fsize)>0 and not listonly):
5783 pyhascontents = True;
5784 if(int(fsize)>0 and listonly):
5785 fcontents = BytesIO();
5786 pyhascontents = False;
5787 fcontents.seek(0, 0);
5788 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5789 fileidnum = fileidnum + 1;
5790 return catlist;
5791
5792 if(not rarfile_support):
5793 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5794 return False;
5795
5796 if(rarfile_support):
5797 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5798 formatspecs = FormatSpecsListToDict(formatspecs);
5799 curinode = 0;
5800 curfid = 0;
5801 inodelist = [];
5802 inodetofile = {};
5803 filetoinode = {};
5804 inodetocatinode = {};
5805 fileidnum = 0;
5806 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5807 return False;
5808 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5809 return False;
5810 rarfp = rarfile.RarFile(infile, "r");
5811 rartest = rarfp.testrar();
5812 if(rartest):
5813 VerbosePrintOut("Bad file found!");
5814 fnumfiles = int(len(rarfp.infolist()));
5815 catver = formatspecs['format_ver'];
5816 fileheaderver = str(int(catver.replace(".", "")));
5817 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5818 catversion = re.findall(r"([\d]+)", fileheader);
5819 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5820 fnumfileshex = format(int(fnumfiles), 'x').lower();
5821 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5822 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5823 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5824 fheadtell = len(fileheader);
5825 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5826 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5827 catfhstart = fheadtell;
5828 is_unix = False;
5829 is_windows = False;
5830 if(member.host_os==rarfile.RAR_OS_UNIX):
5831 is_windows = False;
5832 try:
5833 member.external_attr
5834 is_unix = True;
5835 except AttributeError:
5836 is_unix = False;
5837 elif(member.host_os==rarfile.RAR_OS_WIN32):
5838 is_unix = False;
5839 try:
5840 member.external_attr
5841 is_windows = True;
5842 except AttributeError:
5843 is_windows = False;
5844 else:
5845 is_unix = False;
5846 is_windows = False;
5847 if(re.findall(r"^[.|/]", member.filename)):
5848 fname = member.filename;
5849 else:
5850 fname = "./"+member.filename;
5851 rarinfo = rarfp.getinfo(member.filename);
5852 if(verbose):
5853 VerbosePrintOut(fname);
5854 if(is_unix and member.external_attr !=0):
5855 fpremode = int(member.external_attr);
5856 elif(member.is_file()):
5857 fpremode = stat.S_IFREG + 438;
5858 elif(member.is_symlink()):
5859 fpremode = stat.S_IFLNK + 438;
5860 elif(member.is_dir()):
5861 fpremode = stat.S_IFDIR + 511;
5862 if(is_windows and member.external_attr !=0):
5863 fwinattributes = int(member.external_attr);
5864 else:
5865 fwinattributes = int(0);
5866 fcompression = "";
5867 fcsize = 0;
5868 flinkcount = 0;
5869 ftype = 0;
5870 if(member.is_file()):
5871 ftype = 0;
5872 elif(member.is_symlink()):
5873 ftype = 2;
5874 elif(member.is_dir()):
5875 ftype = 5;
5876 flinkname = "";
5877 if(ftype==2):
5878 flinkname = rarfp.read(member.filename).decode("UTF-8");
5879 fbasedir = os.path.dirname(fname);
5880 fcurfid = curfid;
5881 fcurinode = curfid;
5882 finode = fcurinode;
5883 curfid = curfid + 1;
5884 fdev_minor = 0;
5885 fdev_major = 0;
5886 frdev_minor = 0;
5887 frdev_major = 0;
5888 if(ftype==5):
5889 fsize = "0";
5890 if(ftype==0):
5891 fsize = member.file_size;
5892 try:
5893 if(member.atime):
5894 fatime = int(member.atime.timestamp());
5895 else:
5896 fatime = int(member.mtime.timestamp());
5897 except AttributeError:
5898 fatime = int(member.mtime.timestamp());
5899 fmtime = int(member.mtime.timestamp());
5900 try:
5901 if(member.ctime):
5902 fctime = int(member.ctime.timestamp());
5903 else:
5904 fctime = int(member.mtime.timestamp());
5905 except AttributeError:
5906 fctime = int(member.mtime.timestamp());
5907 fbtime = int(member.mtime.timestamp());
5908 if(is_unix and member.external_attr !=0):
5909 fmode = int(member.external_attr);
5910 fchmode = int(stat.S_IMODE(member.external_attr));
5911 ftypemod = int(stat.S_IFMT(member.external_attr));
5912 elif(member.is_file()):
5913 fmode = int(stat.S_IFREG + 438)
5914 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5915 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5916 elif(member.is_symlink()):
5917 fmode = int(stat.S_IFLNK + 438)
5918 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5919 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5920 elif(member.is_dir()):
5921 fmode = int(stat.S_IFDIR + 511)
5922 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5923 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5924 try:
5925 fuid = os.getuid();
5926 except AttributeError:
5927 fuid = 0;
5928 except KeyError:
5929 fuid = 0;
5930 try:
5931 fgid = os.getgid();
5932 except AttributeError:
5933 fgid = 0;
5934 except KeyError:
5935 fgid = 0;
5936 try:
5937 import pwd;
5938 try:
5939 userinfo = pwd.getpwuid(os.getuid());
5940 funame = userinfo.pw_name;
5941 except KeyError:
5942 funame = "";
5943 except AttributeError:
5944 funame = "";
5945 except ImportError:
5946 funame = "";
5947 fgname = "";
5948 try:
5949 import grp;
5950 try:
5951 groupinfo = grp.getgrgid(os.getgid());
5952 fgname = groupinfo.gr_name;
5953 except KeyError:
5954 fgname = "";
5955 except AttributeError:
5956 fgname = "";
5957 except ImportError:
5958 fgname = "";
5959 fcontents = BytesIO();
5960 if(ftype==0):
5961 fcontents.write(rarfp.read(member.filename));
5962 fcontents.seek(0, 0);
5963 ftypehex = format(ftype, 'x').lower();
5964 extrafields = len(extradata);
5965 extrafieldslist = extradata;
5966 catfextrafields = extrafields;
5967 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5968 if(len(extradata)>0):
5969 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5970 extrasizelen = len(extrasizestr);
5971 extrasizelenhex = format(extrasizelen, 'x').lower();
5972 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5973 catoutlen = len(catoutlist) + len(extradata) + 3;
5974 catoutlenhex = format(catoutlen, 'x').lower();
5975 catoutlist.insert(0, catoutlenhex);
5976 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5977 if(len(extradata)>0):
5978 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5979 if(fsize==0):
5980 checksumlist = [checksumtype, "none"];
5981 else:
5982 checksumlist = [checksumtype, checksumtype];
5983 ccatfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5984 catfnumfields = 24 + catfextrafields;
5985 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5986 fcontents.seek(0, 0);
5987 if(fsize==0):
5988 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5989 else:
5990 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5991 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5992 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5993 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5994 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5995 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5996 catheaderdata = catoutlist;
5997 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5998 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5999 catfcontentstart = fheadtell;
6000 fheadtell += len(catfileoutstr) + 1;
6001 catfcontentend = fheadtell - 1;
6002 catfhend = catfcontentend;
6003 fcontents.seek(0, 0);
6004 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6005 pyhascontents = False;
6006 if(int(fsize)>0 and not listonly):
6007 pyhascontents = True;
6008 if(int(fsize)>0 and listonly):
6009 fcontents = BytesIO();
6010 pyhascontents = False;
6011 fcontents.seek(0, 0);
6012 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6013 fileidnum = fileidnum + 1;
6014 return catlist;
6015
6016 if(not py7zr_support):
6017 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6018 return False;
6019
6020 if(py7zr_support):
6021 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6022 formatspecs = FormatSpecsListToDict(formatspecs);
6023 curinode = 0;
6024 curfid = 0;
6025 inodelist = [];
6026 inodetofile = {};
6027 filetoinode = {};
6028 inodetocatinode = {};
6029 fileidnum = 0;
6030 szpfp = py7zr.SevenZipFile(infile, mode="r");
6031 file_content = szpfp.readall();
6032 #sztest = szpfp.testzip();
6033 sztestalt = szpfp.test();
6034 if(sztestalt):
6035 VerbosePrintOut("Bad file found!");
6036 numfiles = int(len(szpfp.list()));
6037 catver = formatspecs['format_ver'];
6038 fileheaderver = str(int(catver.replace(".", "")));
6039 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6040 catversion = re.findall(r"([\d]+)", fileheader);
6041 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6042 fnumfileshex = format(int(fnumfiles), 'x').lower();
6043 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6044 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6045 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6046 fheadtell = len(fileheader);
6047 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6048 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6049 catfhstart = fheadtell;
6050 if(re.findall(r"^[.|/]", member.filename)):
6051 fname = member.filename;
6052 else:
6053 fname = "./"+member.filename;
6054 if(not member.is_directory):
6055 fpremode = int(stat.S_IFREG + 438);
6056 elif(member.is_directory):
6057 fpremode = int(stat.S_IFDIR + 511);
6058 fwinattributes = int(0);
6059 fcompression = "";
6060 fcsize = 0;
6061 flinkcount = 0;
6062 ftype = 0;
6063 if(member.is_directory):
6064 ftype = 5;
6065 else:
6066 ftype = 0;
6067 flinkname = "";
6068 fbasedir = os.path.dirname(fname);
6069 fcurfid = curfid;
6070 fcurinode = curfid;
6071 finode = fcurinode;
6072 curfid = curfid + 1;
6073 fdev_minor = 0;
6074 fdev_major = 0;
6075 frdev_minor = 0;
6076 frdev_major = 0;
6077 if(ftype==5):
6078 fsize = "0";
6079 fatime = int(member.creationtime.timestamp());
6080 fmtime = int(member.creationtime.timestamp());
6081 fctime = int(member.creationtime.timestamp());
6082 fbtime = int(member.creationtime.timestamp());
6083 if(member.is_directory):
6084 fmode = int(stat.S_IFDIR + 511)
6085 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6086 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6087 else:
6088 fmode = int(stat.S_IFLNK + 438)
6089 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6090 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6091 try:
6092 fuid = os.getuid();
6093 except AttributeError:
6094 fuid = 0;
6095 except KeyError:
6096 fuid = 0;
6097 try:
6098 fgid = os.getgid();
6099 except AttributeError:
6100 fgid = 0;
6101 except KeyError:
6102 fgid = 0;
6103 try:
6104 import pwd;
6105 try:
6106 userinfo = pwd.getpwuid(os.getuid());
6107 funame = userinfo.pw_name;
6108 except KeyError:
6109 funame = "";
6110 except AttributeError:
6111 funame = "";
6112 except ImportError:
6113 funame = "";
6114 fgname = "";
6115 try:
6116 import grp;
6117 try:
6118 groupinfo = grp.getgrgid(os.getgid());
6119 fgname = groupinfo.gr_name;
6120 except KeyError:
6121 fgname = "";
6122 except AttributeError:
6123 fgname = "";
6124 except ImportError:
6125 fgname = "";
6126 fcontents = BytesIO();
6127 if(ftype==0):
6128 fcontents.write(file_content[member.filename].read());
6129 fsize = format(fcontents.tell(), 'x').lower();
6130 fileop.close();
6131 fcontents.seek(0, 0);
6132 ftypehex = format(ftype, 'x').lower();
6133 extrafields = len(extradata);
6134 extrafieldslist = extradata;
6135 catfextrafields = extrafields;
6136 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6137 if(len(extradata)>0):
6138 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6139 extrasizelen = len(extrasizestr);
6140 extrasizelenhex = format(extrasizelen, 'x').lower();
6141 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6142 catoutlen = len(catoutlist) + len(extradata) + 3;
6143 catoutlenhex = format(catoutlen, 'x').lower();
6144 catoutlist.insert(0, catoutlenhex);
6145 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6146 catheaderdata = catoutlist;
6147 if(len(extradata)>0):
6148 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6149 if(fsize==0):
6150 checksumlist = [checksumtype, "none"];
6151 else:
6152 checksumlist = [checksumtype, checksumtype];
6153 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6154 catfnumfields = 24 + catfextrafields;
6155 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6156 fcontents.seek(0, 0);
6157 if(fsize==0):
6158 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6159 else:
6160 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6161 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6162 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6163 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6164 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6165 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6166 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6167 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6168 catfcontentstart = fheadtell;
6169 fheadtell += len(catfileoutstr) + 1;
6170 catfcontentend = fheadtell - 1;
6171 catfhend = catfcontentend;
6172 fcontents.seek(0, 0);
6173 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6174 pyhascontents = False;
6175 if(int(fsize)>0 and not listonly):
6176 pyhascontents = True;
6177 if(int(fsize)>0 and listonly):
6178 fcontents = BytesIO();
6179 pyhascontents = False;
6180 fcontents.seek(0, 0);
6181 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6182 fileidnum = fileidnum + 1;
6183 return catlist;
6184
6185 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6186 formatspecs = FormatSpecsListToDict(formatspecs);
6187 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6188 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
6189 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6190 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
6191 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6192 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
6193 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6194 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
6195 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6196 elif(checkcompressfile=="catfile"):
6197 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6198 else:
6199 return False;
6200 return False;
6201
6202 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6203 formatspecs = FormatSpecsListToDict(formatspecs);
6204 outarray = BytesIO();
6205 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6206 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6207 return listcatfiles;
6208
6209 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6210 formatspecs = FormatSpecsListToDict(formatspecs);
6211 if(isinstance(inarray, dict)):
6212 listcatfiles = inarray;
6213 else:
6214 return False;
6215 if(not listcatfiles):
6216 return False;
6217 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6218 if(returnfp):
6219 catarray.update({'catfp': listcatfiles['catfp']});
6220 lenlist = len(listcatfiles['ffilelist']);
6221 lcfi = 0;
6222 lcfx = int(listcatfiles['fnumfiles']);
6223 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6224 lcfx = int(lenlist);
6225 else:
6226 lcfx = int(listcatfiles['fnumfiles']);
6227 while(lcfi < lcfx):
6228 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6229 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6230 catarray['filetoid'].update(filetoidarray);
6231 catarray['idtofile'].update(idtofilearray);
6232 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6233 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6234 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6235 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6236 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6237 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6238 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6239 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6240 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6241 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6242 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6243 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6244 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6245 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6246 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6247 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6248 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6249 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6250 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6251 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6252 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6253 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6254 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6255 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6256 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6257 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6258 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6259 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6260 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6261 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6262 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6263 lcfi = lcfi + 1;
6264 return catarray;
6265
6266 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6267
6268 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6269 formatspecs = FormatSpecsListToDict(formatspecs);
6270 if(isinstance(infile, dict)):
6271 listcatfiles = infile;
6272 else:
6273 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6274 infile = RemoveWindowsPath(infile);
6275 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6276 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6277 outfile = RemoveWindowsPath(outfile);
6278 checksumtype = checksumtype.lower();
6279 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6280 checksumtype="crc32";
6281 if(checksumtype=="none"):
6282 checksumtype = "";
6283 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6284 compression = "auto";
6285 if(compression not in compressionlist and compression is None):
6286 compression = "auto";
6287 if(verbose):
6288 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6289 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6290 if(os.path.exists(outfile)):
6291 try:
6292 os.unlink(outfile);
6293 except OSError as e:
6294 pass;
6295 if(not listcatfiles):
6296 return False;
6297 if(outfile=="-"):
6298 verbose = False;
6299 catfp = BytesIO();
6300 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6301 catfp = outfile;
6302 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6303 catfp = BytesIO();
6304 else:
6305 fbasename = os.path.splitext(outfile)[0];
6306 fextname = os.path.splitext(outfile)[1];
6307 if(not compresswholefile and fextname in outextlistwd):
6308 compresswholefile = True;
6309 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6310 catver = formatspecs['format_ver'];
6311 fileheaderver = str(int(catver.replace(".", "")));
6312 lenlist = len(listcatfiles['ffilelist']);
6313 fnumfiles = int(listcatfiles['fnumfiles']);
6314 if(lenlist>fnumfiles or lenlist<fnumfiles):
6315 fnumfiles = lenlist;
6316 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6317 lenlist = len(listcatfiles['ffilelist']);
6318 fnumfiles = int(listcatfiles['fnumfiles']);
6319 lcfi = 0;
6320 lcfx = int(listcatfiles['fnumfiles']);
6321 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6322 lcfx = int(lenlist);
6323 else:
6324 lcfx = int(listcatfiles['fnumfiles']);
6325 curinode = 0;
6326 curfid = 0;
6327 inodelist = [];
6328 inodetofile = {};
6329 filetoinode = {};
6330 reallcfi = 0;
6331 while(lcfi < lcfx):
6332 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6333 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6334 else:
6335 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6336 if(verbose):
6337 VerbosePrintOut(fname);
6338 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6339 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6340 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6341 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6342 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6343 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6344 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6345 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6346 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6347 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6348 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6349 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6350 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6351 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6352 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6353 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6354 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6355 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6356 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6357 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6358 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6359 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6360 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6361 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6362 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6363 if(not followlink and len(extradata)<0):
6364 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6365 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6366 fcompression = "";
6367 fcsize = format(int(0), 'x').lower();
6368 if(not compresswholefile):
6369 fcontents.seek(0, 2);
6370 ucfsize = fcontents.tell();
6371 fcontents.seek(0, 0);
6372 if(compression=="auto"):
6373 ilsize = len(compressionlistalt);
6374 ilmin = 0;
6375 ilcsize = [];
6376 while(ilmin < ilsize):
6377 cfcontents = BytesIO();
6378 shutil.copyfileobj(fcontents, cfcontents);
6379 fcontents.seek(0, 0);
6380 cfcontents.seek(0, 0);
6381 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6382 if(cfcontents):
6383 cfcontents.seek(0, 2);
6384 ilcsize.append(cfcontents.tell());
6385 cfcontents.close();
6386 else:
6387 try:
6388 ilcsize.append(sys.maxint);
6389 except AttributeError:
6390 ilcsize.append(sys.maxsize);
6391 ilmin = ilmin + 1;
6392 ilcmin = ilcsize.index(min(ilcsize));
6393 compression = compressionlistalt[ilcmin];
6394 fcontents.seek(0, 0);
6395 cfcontents = BytesIO();
6396 shutil.copyfileobj(fcontents, cfcontents);
6397 cfcontents.seek(0, 0);
6398 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6399 cfcontents.seek(0, 2);
6400 cfsize = cfcontents.tell();
6401 if(ucfsize > cfsize):
6402 fcsize = format(int(cfsize), 'x').lower();
6403 fcompression = compression;
6404 fcontents.close();
6405 fcontents = cfcontents;
6406 if(followlink):
6407 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6408 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6409 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6410 flinkinfo = listcatfiles['ffilelist'][flinkid];
6411 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6412 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6413 flinkname = flinkinfo['flinkname'];
6414 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6415 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6416 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6417 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6418 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6419 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6420 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6421 funame = flinkinfo['funame'];
6422 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6423 fgname = flinkinfo['fgname'];
6424 finode = format(int(flinkinfo['finode']), 'x').lower();
6425 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6426 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6427 fcompression = flinkinfo['fcompression'];
6428 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6429 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6430 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6431 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6432 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6433 fseeknextfile = flinkinfo['fseeknextfile'];
6434 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6435 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6436 if(len(extradata)<0):
6437 extradata = flinkinfo['fextralist'];
6438 fcontents = flinkinfo['fcontents'];
6439 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6440 else:
6441 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6442 fcurfid = format(curfid, 'x').lower();
6443 if(not followlink and finode!=0):
6444 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6445 fcurinode = format(int(curinode), 'x').lower();
6446 inodetofile.update({curinode: fname});
6447 filetoinode.update({fname: curinode});
6448 curinode = curinode + 1;
6449 else:
6450 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6451 else:
6452 fcurinode = format(int(curinode), 'x').lower();
6453 curinode = curinode + 1;
6454 curfid = curfid + 1;
6455 if(fcompression=="none"):
6456 fcompression = "";
6457 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6458 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6459 fcontents.close();
6460 lcfi = lcfi + 1;
6461 reallcfi = reallcfi + 1;
6462 if(lcfx>0):
6463 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6464 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6465 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6466 try:
6467 catfp.flush();
6468 os.fsync(catfp.fileno());
6469 except io.UnsupportedOperation:
6470 pass;
6471 except AttributeError:
6472 pass;
6473 except OSError as e:
6474 pass;
6475 if(outfile=="-"):
6476 catfp.seek(0, 0);
6477 if(hasattr(sys.stdout, "buffer")):
6478 shutil.copyfileobj(catfp, sys.stdout.buffer);
6479 else:
6480 shutil.copyfileobj(catfp, sys.stdout);
6481 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6482 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6483 catfp.seek(0, 0);
6484 upload_file_to_internet_file(catfp, outfile);
6485 if(returnfp):
6486 catfp.seek(0, 0);
6487 return catfp;
6488 else:
6489 catfp.close();
6490 return True;
6491
6492 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6493
6494 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6495 formatspecs = FormatSpecsListToDict(formatspecs);
6496 catfp = BytesIO(catstr);
6497 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6498 return listcatfiles;
6499
6500 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6501
6502 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6503 formatspecs = FormatSpecsListToDict(formatspecs);
6504 outarray = BytesIO();
6505 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6506 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6507 return listcatfiles;
6508
6509 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6510
6511 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6512 formatspecs = FormatSpecsListToDict(formatspecs);
6513 if(outdir is not None):
6514 outdir = RemoveWindowsPath(outdir);
6515 if(verbose):
6516 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6517 if(isinstance(infile, dict)):
6518 listcatfiles = infile;
6519 else:
6520 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6521 infile = RemoveWindowsPath(infile);
6522 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6523 if(not listcatfiles):
6524 return False;
6525 lenlist = len(listcatfiles['ffilelist']);
6526 fnumfiles = int(listcatfiles['fnumfiles']);
6527 lcfi = 0;
6528 lcfx = int(listcatfiles['fnumfiles']);
6529 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6530 lcfx = int(lenlist);
6531 else:
6532 lcfx = int(listcatfiles['fnumfiles']);
6533 while(lcfi<lcfx):
6534 funame = "";
6535 try:
6536 import pwd;
6537 try:
6538 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6539 funame = userinfo.pw_name;
6540 except KeyError:
6541 funame = "";
6542 except ImportError:
6543 funame = "";
6544 fgname = "";
6545 try:
6546 import grp;
6547 try:
6548 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6549 fgname = groupinfo.gr_name;
6550 except KeyError:
6551 fgname = "";
6552 except ImportError:
6553 fgname = "";
6554 if(verbose):
6555 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6556 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6557 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6558 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6559 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6560 try:
6561 fpc.flush();
6562 os.fsync(fpc.fileno());
6563 except io.UnsupportedOperation:
6564 pass;
6565 except AttributeError:
6566 pass;
6567 except OSError as e:
6568 pass;
6569 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6570 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6571 if(preservepermissions):
6572 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6573 if(preservetime):
6574 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6575 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6576 if(followlink):
6577 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6578 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6579 flinkinfo = listcatfiles['ffilelist'][flinkid];
6580 funame = "";
6581 try:
6582 import pwd;
6583 try:
6584 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6585 funame = userinfo.pw_name;
6586 except KeyError:
6587 funame = "";
6588 except ImportError:
6589 funame = "";
6590 fgname = "";
6591 try:
6592 import grp;
6593 try:
6594 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6595 fgname = groupinfo.gr_name;
6596 except KeyError:
6597 fgname = "";
6598 except ImportError:
6599 fgname = "";
6600 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6601 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6602 flinkinfo['fcontents'].seek(0, 0);
6603 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6604 try:
6605 fpc.flush();
6606 os.fsync(fpc.fileno());
6607 except io.UnsupportedOperation:
6608 pass;
6609 except AttributeError:
6610 pass;
6611 except OSError as e:
6612 pass;
6613 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6614 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6615 if(preservepermissions):
6616 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6617 if(preservetime):
6618 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6619 if(flinkinfo['ftype']==1):
6620 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6621 if(flinkinfo['ftype']==2):
6622 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6623 if(flinkinfo['ftype']==5):
6624 if(preservepermissions):
6625 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6626 else:
6627 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6628 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6629 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6630 if(preservepermissions):
6631 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6632 if(preservetime):
6633 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6634 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6635 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6636 else:
6637 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6638 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6639 if(followlink):
6640 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6641 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6642 flinkinfo = listcatfiles['ffilelist'][flinkid];
6643 funame = "";
6644 try:
6645 import pwd;
6646 try:
6647 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6648 funame = userinfo.pw_name;
6649 except KeyError:
6650 funame = "";
6651 except ImportError:
6652 funame = "";
6653 fgname = "";
6654 try:
6655 import grp;
6656 try:
6657 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6658 fgname = groupinfo.gr_name;
6659 except KeyError:
6660 fgname = "";
6661 except ImportError:
6662 fgname = "";
6663 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6664 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6665 flinkinfo['fcontents'].seek(0, 0);
6666 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6667 try:
6668 fpc.flush();
6669 os.fsync(fpc.fileno());
6670 except io.UnsupportedOperation:
6671 pass;
6672 except AttributeError:
6673 pass;
6674 except OSError as e:
6675 pass;
6676 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6677 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6678 if(preservepermissions):
6679 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6680 if(preservetime):
6681 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6682 if(flinkinfo['ftype']==1):
6683 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6684 if(flinkinfo['ftype']==2):
6685 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6686 if(flinkinfo['ftype']==5):
6687 if(preservepermissions):
6688 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6689 else:
6690 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6691 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6692 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6693 if(preservepermissions):
6694 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6695 if(preservetime):
6696 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6697 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6698 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6699 else:
6700 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6701 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6702 if(preservepermissions):
6703 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6704 else:
6705 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6706 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6707 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6708 if(preservepermissions):
6709 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6710 if(preservetime):
6711 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6712 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6713 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6714 lcfi = lcfi + 1;
6715 if(returnfp):
6716 return listcatfiles['ffilelist']['catfp'];
6717 else:
6718 return True;
6719
6720 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6721
6722 if(hasattr(shutil, "register_unpack_format")):
6723 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6724 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6725 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6726
6727 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6728 formatspecs = FormatSpecsListToDict(formatspecs);
6729 catfp = BytesIO(catstr);
6730 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6731 return listcatfiles;
6732
6733 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6734
6735 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6736 formatspecs = FormatSpecsListToDict(formatspecs);
6737 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6738 if(isinstance(infile, dict)):
6739 listcatfiles = infile;
6740 else:
6741 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6742 infile = RemoveWindowsPath(infile);
6743 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6744 if(not listcatfiles):
6745 return False;
6746 lenlist = len(listcatfiles['ffilelist']);
6747 fnumfiles = int(listcatfiles['fnumfiles']);
6748 lcfi = 0;
6749 lcfx = int(listcatfiles['fnumfiles']);
6750 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6751 lcfx = int(lenlist);
6752 else:
6753 lcfx = int(listcatfiles['fnumfiles']);
6754 returnval = {};
6755 while(lcfi<lcfx):
6756 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6757 if(not verbose):
6758 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6759 if(verbose):
6760 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6761 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6762 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6763 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6764 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6765 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6766 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6767 if(len(fuprint)<=0):
6768 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6769 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6770 if(len(fgprint)<=0):
6771 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6772 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6773 lcfi = lcfi + 1;
6774 if(returnfp):
6775 return listcatfiles['catfp'];
6776 else:
6777 return True;
6778
6779 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6780
6781 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6782 formatspecs = FormatSpecsListToDict(formatspecs);
6783 catfp = BytesIO(catstr);
6784 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6785 return listcatfiles;
6786
6787 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6788
6789 def TarFileListFiles(infile, verbose=False, returnfp=False):
6790 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6791 if(infile=="-"):
6792 infile = BytesIO();
6793 if(hasattr(sys.stdin, "buffer")):
6794 shutil.copyfileobj(sys.stdin.buffer, infile);
6795 else:
6796 shutil.copyfileobj(sys.stdin, infile);
6797 infile.seek(0, 0);
6798 if(not infile):
6799 return False;
6800 infile.seek(0, 0);
6801 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6802 infile = download_file_from_internet_file(infile);
6803 infile.seek(0, 0);
6804 if(not infile):
6805 return False;
6806 infile.seek(0, 0);
6807 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6808 return False;
6809 elif(os.path.exists(infile) and os.path.isfile(infile)):
6810 try:
6811 if(not tarfile.TarFileCheck(infile)):
6812 return False;
6813 except AttributeError:
6814 if(not TarFileCheck(infile)):
6815 return False;
6816 else:
6817 return False;
6818 try:
6819 if(hasattr(infile, "read") or hasattr(infile, "write")):
6820 tarfp = tarfile.open(fileobj=infile, mode="r");
6821 else:
6822 tarfp = tarfile.open(infile, "r");
6823 except FileNotFoundError:
6824 return False;
6825 lcfi = 0
6826 returnval = {};
6827 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6828 returnval.update({lcfi: member.name});
6829 fpremode = member.mode;
6830 ffullmode = member.mode;
6831 flinkcount = 0;
6832 ftype = 0;
6833 if(member.isreg()):
6834 ffullmode = member.mode + stat.S_IFREG;
6835 ftype = 0;
6836 elif(member.isdev()):
6837 ffullmode = member.mode;
6838 ftype = 7;
6839 elif(member.islnk()):
6840 ffullmode = member.mode + stat.S_IFREG;
6841 ftype = 1;
6842 elif(member.issym()):
6843 ffullmode = member.mode + stat.S_IFLNK;
6844 ftype = 2;
6845 elif(member.ischr()):
6846 ffullmode = member.mode + stat.S_IFCHR;
6847 ftype = 3;
6848 elif(member.isblk()):
6849 ffullmode = member.mode + stat.S_IFBLK;
6850 ftype = 4;
6851 elif(member.isdir()):
6852 ffullmode = member.mode + stat.S_IFDIR;
6853 ftype = 5;
6854 elif(member.isfifo()):
6855 ffullmode = member.mode + stat.S_IFIFO;
6856 ftype = 6;
6857 elif(member.issparse()):
6858 ffullmode = member.mode;
6859 ftype = 12;
6860 if(not verbose):
6861 VerbosePrintOut(member.name);
6862 elif(verbose):
6863 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6864 printfname = member.name;
6865 if(member.islnk()):
6866 printfname = member.name + " link to " + member.linkname;
6867 elif(member.issym()):
6868 printfname = member.name + " -> " + member.linkname;
6869 fuprint = member.uname;
6870 if(len(fuprint)<=0):
6871 fuprint = member.uid;
6872 fgprint = member.gname;
6873 if(len(fgprint)<=0):
6874 fgprint = member.gid;
6875 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6876 lcfi = lcfi + 1;
6877 if(returnfp):
6878 return listcatfiles['catfp'];
6879 else:
6880 return True;
6881
6882 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6883 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6884 if(infile=="-"):
6885 infile = BytesIO();
6886 if(hasattr(sys.stdin, "buffer")):
6887 shutil.copyfileobj(sys.stdin.buffer, infile);
6888 else:
6889 shutil.copyfileobj(sys.stdin, infile);
6890 infile.seek(0, 0);
6891 if(not infile):
6892 return False;
6893 infile.seek(0, 0);
6894 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6895 infile = download_file_from_internet_file(infile);
6896 infile.seek(0, 0);
6897 if(not infile):
6898 return False;
6899 infile.seek(0, 0);
6900 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6901 return False;
6902 else:
6903 return False;
6904 if(not zipfile.is_zipfile(infile)):
6905 return False;
6906 try:
6907 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6908 except FileNotFoundError:
6909 return False;
6910 lcfi = 0;
6911 returnval = {};
6912 ziptest = zipfp.testzip();
6913 if(ziptest):
6914 VerbosePrintOut("Bad file found!");
6915 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6916 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6917 fwinattributes = int(zipinfo.external_attr);
6918 if(not member.is_dir()):
6919 fmode = int(stat.S_IFREG + 438);
6920 fchmode = int(stat.S_IMODE(fmode));
6921 ftypemod = int(stat.S_IFMT(fmode));
6922 elif(member.is_dir()):
6923 fmode = int(stat.S_IFDIR + 511);
6924 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6925 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6926 elif(zipinfo.create_system==3):
6927 fwinattributes =int(0);
6928 fmode = int(zipinfo.external_attr);
6929 fchmode = int(stat.S_IMODE(fmode));
6930 ftypemod = int(stat.S_IFMT(fmode));
6931 else:
6932 fwinattributes = int(0);
6933 if(not member.is_dir()):
6934 fmode = int(stat.S_IFREG + 438);
6935 fchmode = int(stat.S_IMODE(fmode));
6936 ftypemod = int(stat.S_IFMT(fmode));
6937 elif(member.is_dir()):
6938 fmode = int(stat.S_IFDIR + 511);
6939 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6940 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6941 returnval.update({lcfi: member.filename});
6942 if(not verbose):
6943 VerbosePrintOut(member.filename);
6944 if(verbose):
6945 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6946 permissionstr = "";
6947 for fmodval in str(oct(fmode))[-3:]:
6948 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6949 if(not member.is_dir()):
6950 ftype = 0;
6951 permissionstr = "-" + permissionstr;
6952 elif(member.is_dir()):
6953 ftype = 5;
6954 permissionstr = "d" + permissionstr;
6955 printfname = member.filename;
6956 try:
6957 fuid = int(os.getuid());
6958 except AttributeError:
6959 fuid = int(0);
6960 except KeyError:
6961 fuid = int(0);
6962 try:
6963 fgid = int(os.getgid());
6964 except AttributeError:
6965 fgid = int(0);
6966 except KeyError:
6967 fgid = int(0);
6968 try:
6969 import pwd;
6970 try:
6971 userinfo = pwd.getpwuid(os.getuid());
6972 funame = userinfo.pw_name;
6973 except KeyError:
6974 funame = "";
6975 except AttributeError:
6976 funame = "";
6977 except ImportError:
6978 funame = "";
6979 fgname = "";
6980 try:
6981 import grp;
6982 try:
6983 groupinfo = grp.getgrgid(os.getgid());
6984 fgname = groupinfo.gr_name;
6985 except KeyError:
6986 fgname = "";
6987 except AttributeError:
6988 fgname = "";
6989 except ImportError:
6990 fgname = "";
6991 fuprint = funame;
6992 if(len(fuprint)<=0):
6993 fuprint = str(fuid);
6994 fgprint = fgname;
6995 if(len(fgprint)<=0):
6996 fgprint = str(fgid);
6997 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6998 lcfi = lcfi + 1;
6999 if(returnfp):
7000 return listcatfiles['catfp'];
7001 else:
7002 return True;
7003
7004 if(not rarfile_support):
7005 def RarFileListFiles(infile, verbose=False, returnfp=False):
7006 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7007 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7008 return False;
7009
7010 if(rarfile_support):
7011 def RarFileListFiles(infile, verbose=False, returnfp=False):
7012 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7013 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7014 return False;
7015 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7016 return False;
7017 lcfi = 0;
7018 returnval = {};
7019 rarfp = rarfile.RarFile(infile, "r");
7020 rartest = rarfp.testrar();
7021 if(rartest):
7022 VerbosePrintOut("Bad file found!");
7023 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7024 is_unix = False;
7025 is_windows = False;
7026 if(member.host_os==rarfile.RAR_OS_UNIX):
7027 is_windows = False;
7028 try:
7029 member.external_attr
7030 is_unix = True;
7031 except AttributeError:
7032 is_unix = False;
7033 elif(member.host_os==rarfile.RAR_OS_WIN32):
7034 is_unix = False;
7035 try:
7036 member.external_attr
7037 is_windows = True;
7038 except AttributeError:
7039 is_windows = False;
7040 else:
7041 is_unix = False;
7042 is_windows = False;
7043 if(is_unix and member.external_attr !=0):
7044 fpremode = int(member.external_attr);
7045 elif(member.is_file()):
7046 fpremode = int(stat.S_IFREG + 438);
7047 elif(member.is_symlink()):
7048 fpremode = int(stat.S_IFLNK + 438);
7049 elif(member.is_dir()):
7050 fpremode = int(stat.S_IFDIR + 511);
7051 if(is_windows and member.external_attr !=0):
7052 fwinattributes = int(member.external_attr);
7053 else:
7054 fwinattributes = int(0);
7055 if(is_unix and member.external_attr !=0):
7056 fmode = int(member.external_attr);
7057 fchmode = int(stat.S_IMODE(member.external_attr));
7058 ftypemod = int(stat.S_IFMT(member.external_attr));
7059 elif(member.is_file()):
7060 fmode = int(stat.S_IFREG + 438);
7061 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7062 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7063 elif(member.is_symlink()):
7064 fmode = int(stat.S_IFLNK + 438);
7065 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7066 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7067 elif(member.is_dir()):
7068 fmode = int(stat.S_IFDIR + 511);
7069 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7070 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7071 returnval.update({lcfi: member.filename});
7072 if(not verbose):
7073 VerbosePrintOut(member.filename);
7074 if(verbose):
7075 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7076 permissionstr = "";
7077 for fmodval in str(oct(fmode))[-3:]:
7078 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7079 if(member.is_file()):
7080 ftype = 0;
7081 permissionstr = "-" + permissionstr;
7082 printfname = member.filename;
7083 elif(member.is_symlink()):
7084 ftype = 2;
7085 permissionstr = "l" + permissionstr;
7086 printfname = member.name + " -> " + member.read().decode("UTF-8");
7087 elif(member.is_dir()):
7088 ftype = 5;
7089 permissionstr = "d" + permissionstr;
7090 printfname = member.filename;
7091 try:
7092 fuid = int(os.getuid());
7093 except AttributeError:
7094 fuid = int(0);
7095 except KeyError:
7096 fuid = int(0);
7097 try:
7098 fgid = int(os.getgid());
7099 except AttributeError:
7100 fgid = int(0);
7101 except KeyError:
7102 fgid = int(0);
7103 try:
7104 import pwd;
7105 try:
7106 userinfo = pwd.getpwuid(os.getuid());
7107 funame = userinfo.pw_name;
7108 except KeyError:
7109 funame = "";
7110 except AttributeError:
7111 funame = "";
7112 except ImportError:
7113 funame = "";
7114 fgname = "";
7115 try:
7116 import grp;
7117 try:
7118 groupinfo = grp.getgrgid(os.getgid());
7119 fgname = groupinfo.gr_name;
7120 except KeyError:
7121 fgname = "";
7122 except AttributeError:
7123 fgname = "";
7124 except ImportError:
7125 fgname = "";
7126 fuprint = funame;
7127 if(len(fuprint)<=0):
7128 fuprint = str(fuid);
7129 fgprint = fgname;
7130 if(len(fgprint)<=0):
7131 fgprint = str(fgid);
7132 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7133 lcfi = lcfi + 1;
7134 if(returnfp):
7135 return listcatfiles['catfp'];
7136 else:
7137 return True;
7138
7139 if(not py7zr_support):
7140 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7141 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7142 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7143 return False;
7144
7145 if(py7zr_support):
7146 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7147 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7148 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7149 return False;
7150 lcfi = 0;
7151 returnval = {};
7152 szpfp = py7zr.SevenZipFile(infile, mode="r");
7153 file_content = szpfp.readall();
7154 #sztest = szpfp.testzip();
7155 sztestalt = szpfp.test();
7156 if(sztestalt):
7157 VerbosePrintOut("Bad file found!");
7158 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7159 if(re.findall(r"^[.|/]", member.filename)):
7160 fname = member.filename;
7161 else:
7162 fname = "./"+member.filename;
7163 if(not member.is_directory):
7164 fpremode = int(stat.S_IFREG + 438);
7165 elif(member.is_directory):
7166 fpremode = int(stat.S_IFDIR + 511);
7167 fwinattributes = int(0);
7168 if(member.is_directory):
7169 fmode = int(stat.S_IFDIR + 511);
7170 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7171 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7172 else:
7173 fmode = int(stat.S_IFLNK + 438);
7174 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7175 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7176 returnval.update({lcfi: member.filename});
7177 if(not verbose):
7178 VerbosePrintOut(member.filename);
7179 if(verbose):
7180 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7181 permissionstr = "";
7182 for fmodval in str(oct(fmode))[-3:]:
7183 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7184 fsize = int("0");
7185 if(not member.is_directory):
7186 ftype = 0;
7187 permissionstr = "-" + permissionstr;
7188 printfname = member.filename;
7189 elif(member.is_directory):
7190 ftype = 5;
7191 permissionstr = "d" + permissionstr;
7192 printfname = member.filename;
7193 if(ftype==0):
7194 fsize = len(file_content[member.filename].read());
7195 file_content[member.filename].close();
7196 try:
7197 fuid = int(os.getuid());
7198 except AttributeError:
7199 fuid = int(0);
7200 except KeyError:
7201 fuid = int(0);
7202 try:
7203 fgid = int(os.getgid());
7204 except AttributeError:
7205 fgid = int(0);
7206 except KeyError:
7207 fgid = int(0);
7208 try:
7209 import pwd;
7210 try:
7211 userinfo = pwd.getpwuid(os.getuid());
7212 funame = userinfo.pw_name;
7213 except KeyError:
7214 funame = "";
7215 except AttributeError:
7216 funame = "";
7217 except ImportError:
7218 funame = "";
7219 fgname = "";
7220 try:
7221 import grp;
7222 try:
7223 groupinfo = grp.getgrgid(os.getgid());
7224 fgname = groupinfo.gr_name;
7225 except KeyError:
7226 fgname = "";
7227 except AttributeError:
7228 fgname = "";
7229 except ImportError:
7230 fgname = "";
7231 fuprint = funame;
7232 if(len(fuprint)<=0):
7233 fuprint = str(fuid);
7234 fgprint = fgname;
7235 if(len(fgprint)<=0):
7236 fgprint = str(fgid);
7237 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7238 lcfi = lcfi + 1;
7239 if(returnfp):
7240 return listcatfiles['catfp'];
7241 else:
7242 return True;
7243
7244 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7245 formatspecs = FormatSpecsListToDict(formatspecs);
7246 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7247 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7248 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
7249 return TarFileListFiles(infile, verbose, returnfp);
7250 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
7251 return ZipFileListFiles(infile, verbose, returnfp);
7252 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
7253 return RarFileListFiles(infile, verbose, returnfp);
7254 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
7255 return SevenZipFileListFiles(infile, verbose, returnfp);
7256 elif(checkcompressfile=="catfile"):
7257 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7258 else:
7259 return False;
7260 return False;
7261
7262 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7263 formatspecs = FormatSpecsListToDict(formatspecs);
7264 outarray = BytesIO();
7265 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7266 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7267 return listcatfiles;
7268
7269 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7270 formatspecs = FormatSpecsListToDict(formatspecs);
7271 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7272 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7273 return listcatfiles;
7274
7275 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7276 formatspecs = FormatSpecsListToDict(formatspecs);
7277 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7278 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7279 return listcatfiles;
7280
7281 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7282
7283 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7284 formatspecs = FormatSpecsListToDict(formatspecs);
7285 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7286 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7287 return listcatfiles;
7288
7289 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7290
7291 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7292 formatspecs = FormatSpecsListToDict(formatspecs);
7293 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7294 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7295 return listcatfiles;
7296
7297 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7298
7299 if(not rarfile_support):
7300 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7301 return False;
7302
7303 if(rarfile_support):
7304 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7305 formatspecs = FormatSpecsListToDict(formatspecs);
7306 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7307 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7308 return listcatfiles;
7309
7310 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7311
7312 if(not py7zr_support):
7313 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7314 return False;
7315
7316 if(py7zr_support):
7317 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7318 formatspecs = FormatSpecsListToDict(formatspecs);
7319 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7320 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7321 return listcatfiles;
7322
7323 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7324
7325 def download_file_from_ftp_file(url):
7326 urlparts = urlparse(url);
7327 file_name = os.path.basename(urlparts.path);
7328 file_dir = os.path.dirname(urlparts.path);
7329 if(urlparts.username is not None):
7330 ftp_username = urlparts.username;
7331 else:
7332 ftp_username = "anonymous";
7333 if(urlparts.password is not None):
7334 ftp_password = urlparts.password;
7335 elif(urlparts.password is None and urlparts.username=="anonymous"):
7336 ftp_password = "anonymous";
7337 else:
7338 ftp_password = "";
7339 if(urlparts.scheme=="ftp"):
7340 ftp = FTP();
7341 elif(urlparts.scheme=="ftps" and ftpssl):
7342 ftp = FTP_TLS();
7343 else:
7344 return False;
7345 if(urlparts.scheme=="sftp"):
7346 if(__use_pysftp__):
7347 return download_file_from_pysftp_file(url);
7348 else:
7349 return download_file_from_sftp_file(url);
7350 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7351 return download_file_from_http_file(url);
7352 ftp_port = urlparts.port;
7353 if(urlparts.port is None):
7354 ftp_port = 21;
7355 try:
7356 ftp.connect(urlparts.hostname, ftp_port);
7357 except socket.gaierror:
7358 log.info("Error With URL "+url);
7359 return False;
7360 except socket.timeout:
7361 log.info("Error With URL "+url);
7362 return False;
7363 ftp.login(urlparts.username, urlparts.password);
7364 if(urlparts.scheme=="ftps"):
7365 ftp.prot_p();
7366 ftpfile = BytesIO();
7367 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7368 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7369 ftp.close();
7370 ftpfile.seek(0, 0);
7371 return ftpfile;
7372
7373 def download_file_from_ftp_string(url):
7374 ftpfile = download_file_from_ftp_file(url);
7375 return ftpfile.read();
7376
7377 def upload_file_to_ftp_file(ftpfile, url):
7378 urlparts = urlparse(url);
7379 file_name = os.path.basename(urlparts.path);
7380 file_dir = os.path.dirname(urlparts.path);
7381 if(urlparts.username is not None):
7382 ftp_username = urlparts.username;
7383 else:
7384 ftp_username = "anonymous";
7385 if(urlparts.password is not None):
7386 ftp_password = urlparts.password;
7387 elif(urlparts.password is None and urlparts.username=="anonymous"):
7388 ftp_password = "anonymous";
7389 else:
7390 ftp_password = "";
7391 if(urlparts.scheme=="ftp"):
7392 ftp = FTP();
7393 elif(urlparts.scheme=="ftps" and ftpssl):
7394 ftp = FTP_TLS();
7395 else:
7396 return False;
7397 if(urlparts.scheme=="sftp"):
7398 if(__use_pysftp__):
7399 return upload_file_to_pysftp_file(url);
7400 else:
7401 return upload_file_to_sftp_file(url);
7402 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7403 return False;
7404 ftp_port = urlparts.port;
7405 if(urlparts.port is None):
7406 ftp_port = 21;
7407 try:
7408 ftp.connect(urlparts.hostname, ftp_port);
7409 except socket.gaierror:
7410 log.info("Error With URL "+url);
7411 return False;
7412 except socket.timeout:
7413 log.info("Error With URL "+url);
7414 return False;
7415 ftp.login(urlparts.username, urlparts.password);
7416 if(urlparts.scheme=="ftps"):
7417 ftp.prot_p();
7418 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7419 ftp.close();
7420 ftpfile.seek(0, 0);
7421 return ftpfile;
7422
7423 def upload_file_to_ftp_string(ftpstring, url):
7424 ftpfileo = BytesIO(ftpstring);
7425 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7426 ftpfileo.close();
7427 return ftpfile;
7428
7429 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7430 # Parse the URL to extract username and password if present
7431 urlparts = urlparse(url);
7432 username = urlparts.username;
7433 password = urlparts.password;
7434 # Rebuild the URL without the username and password
7435 netloc = urlparts.hostname;
7436 if(urlparts.scheme=="sftp"):
7437 if(__use_pysftp__):
7438 return download_file_from_pysftp_file(url);
7439 else:
7440 return download_file_from_sftp_file(url);
7441 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7442 return download_file_from_ftp_file(url);
7443 if urlparts.port:
7444 netloc += ':' + str(urlparts.port);
7445 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7446 # Create a temporary file object
7447 httpfile = BytesIO();
7448 if haverequests:
7449 # Use the requests library if available
7450 if username and password:
7451 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7452 else:
7453 response = requests.get(rebuilt_url, headers=headers, stream=True);
7454 response.raw.decode_content = True
7455 shutil.copyfileobj(response.raw, httpfile);
7456 else:
7457 # Build a Request object for urllib
7458 request = Request(rebuilt_url, headers=headers);
7459 # Create an opener object for handling URLs
7460 if username and password:
7461 # Create a password manager
7462 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7463 # Add the username and password
7464 password_mgr.add_password(None, rebuilt_url, username, password);
7465 # Create an authentication handler using the password manager
7466 auth_handler = HTTPBasicAuthHandler(password_mgr);
7467 # Build the opener with the authentication handler
7468 opener = build_opener(auth_handler);
7469 else:
7470 opener = build_opener();
7471 with opener.open(request) as response:
7472 shutil.copyfileobj(response, httpfile);
7473 # Reset file pointer to the start
7474 httpfile.seek(0, 0);
7475 # Return the temporary file object
7476 return httpfile;
7477
7478 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7479 httpfile = download_file_from_http_file(url, headers);
7480 return ftpfile.read();
7481
7482 if(haveparamiko):
7483 def download_file_from_sftp_file(url):
7484 urlparts = urlparse(url);
7485 file_name = os.path.basename(urlparts.path);
7486 file_dir = os.path.dirname(urlparts.path);
7487 sftp_port = urlparts.port;
7488 if(urlparts.port is None):
7489 sftp_port = 22;
7490 else:
7491 sftp_port = urlparts.port;
7492 if(urlparts.username is not None):
7493 sftp_username = urlparts.username;
7494 else:
7495 sftp_username = "anonymous";
7496 if(urlparts.password is not None):
7497 sftp_password = urlparts.password;
7498 elif(urlparts.password is None and urlparts.username=="anonymous"):
7499 sftp_password = "anonymous";
7500 else:
7501 sftp_password = "";
7502 if(urlparts.scheme=="ftp"):
7503 return download_file_from_ftp_file(url);
7504 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7505 return download_file_from_http_file(url);
7506 if(urlparts.scheme!="sftp"):
7507 return False;
7508 ssh = paramiko.SSHClient();
7509 ssh.load_system_host_keys();
7510 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7511 try:
7512 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7513 except paramiko.ssh_exception.SSHException:
7514 return False;
7515 except socket.gaierror:
7516 log.info("Error With URL "+url);
7517 return False;
7518 except socket.timeout:
7519 log.info("Error With URL "+url);
7520 return False;
7521 sftp = ssh.open_sftp();
7522 sftpfile = BytesIO();
7523 sftp.getfo(urlparts.path, sftpfile);
7524 sftp.close();
7525 ssh.close();
7526 sftpfile.seek(0, 0);
7527 return sftpfile;
7528 else:
7529 def download_file_from_sftp_file(url):
7530 return False;
7531
7532 if(haveparamiko):
7533 def download_file_from_sftp_string(url):
7534 sftpfile = download_file_from_sftp_file(url);
7535 return sftpfile.read();
7536 else:
7537 def download_file_from_ftp_string(url):
7538 return False;
7539
7540 if(haveparamiko):
7541 def upload_file_to_sftp_file(sftpfile, url):
7542 urlparts = urlparse(url);
7543 file_name = os.path.basename(urlparts.path);
7544 file_dir = os.path.dirname(urlparts.path);
7545 sftp_port = urlparts.port;
7546 if(urlparts.port is None):
7547 sftp_port = 22;
7548 else:
7549 sftp_port = urlparts.port;
7550 if(urlparts.username is not None):
7551 sftp_username = urlparts.username;
7552 else:
7553 sftp_username = "anonymous";
7554 if(urlparts.password is not None):
7555 sftp_password = urlparts.password;
7556 elif(urlparts.password is None and urlparts.username=="anonymous"):
7557 sftp_password = "anonymous";
7558 else:
7559 sftp_password = "";
7560 if(urlparts.scheme=="ftp"):
7561 return upload_file_to_ftp_file(url);
7562 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7563 return False;
7564 if(urlparts.scheme!="sftp"):
7565 return False;
7566 ssh = paramiko.SSHClient();
7567 ssh.load_system_host_keys();
7568 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7569 try:
7570 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7571 except paramiko.ssh_exception.SSHException:
7572 return False;
7573 except socket.gaierror:
7574 log.info("Error With URL "+url);
7575 return False;
7576 except socket.timeout:
7577 log.info("Error With URL "+url);
7578 return False;
7579 sftp = ssh.open_sftp();
7580 sftp.putfo(sftpfile, urlparts.path);
7581 sftp.close();
7582 ssh.close();
7583 sftpfile.seek(0, 0);
7584 return sftpfile;
7585 else:
7586 def upload_file_to_sftp_file(sftpfile, url):
7587 return False;
7588
7589 if(haveparamiko):
7590 def upload_file_to_sftp_string(sftpstring, url):
7591 sftpfileo = BytesIO(sftpstring);
7592 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7593 sftpfileo.close();
7594 return sftpfile;
7595 else:
7596 def upload_file_to_sftp_string(url):
7597 return False;
7598
7599 if(havepysftp):
7600 def download_file_from_pysftp_file(url):
7601 urlparts = urlparse(url);
7602 file_name = os.path.basename(urlparts.path);
7603 file_dir = os.path.dirname(urlparts.path);
7604 sftp_port = urlparts.port;
7605 if(urlparts.port is None):
7606 sftp_port = 22;
7607 else:
7608 sftp_port = urlparts.port;
7609 if(urlparts.username is not None):
7610 sftp_username = urlparts.username;
7611 else:
7612 sftp_username = "anonymous";
7613 if(urlparts.password is not None):
7614 sftp_password = urlparts.password;
7615 elif(urlparts.password is None and urlparts.username=="anonymous"):
7616 sftp_password = "anonymous";
7617 else:
7618 sftp_password = "";
7619 if(urlparts.scheme=="ftp"):
7620 return download_file_from_ftp_file(url);
7621 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7622 return download_file_from_http_file(url);
7623 if(urlparts.scheme!="sftp"):
7624 return False;
7625 try:
7626 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7627 except paramiko.ssh_exception.SSHException:
7628 return False;
7629 except socket.gaierror:
7630 log.info("Error With URL "+url);
7631 return False;
7632 except socket.timeout:
7633 log.info("Error With URL "+url);
7634 return False;
7635 sftp = ssh.open_sftp();
7636 sftpfile = BytesIO();
7637 sftp.getfo(urlparts.path, sftpfile);
7638 sftp.close();
7639 ssh.close();
7640 sftpfile.seek(0, 0);
7641 return sftpfile;
7642 else:
7643 def download_file_from_pysftp_file(url):
7644 return False;
7645
7646 if(havepysftp):
7647 def download_file_from_pysftp_string(url):
7648 sftpfile = download_file_from_pysftp_file(url);
7649 return sftpfile.read();
7650 else:
7651 def download_file_from_ftp_string(url):
7652 return False;
7653
7654 if(havepysftp):
7655 def upload_file_to_pysftp_file(sftpfile, url):
7656 urlparts = urlparse(url);
7657 file_name = os.path.basename(urlparts.path);
7658 file_dir = os.path.dirname(urlparts.path);
7659 sftp_port = urlparts.port;
7660 if(urlparts.port is None):
7661 sftp_port = 22;
7662 else:
7663 sftp_port = urlparts.port;
7664 if(urlparts.username is not None):
7665 sftp_username = urlparts.username;
7666 else:
7667 sftp_username = "anonymous";
7668 if(urlparts.password is not None):
7669 sftp_password = urlparts.password;
7670 elif(urlparts.password is None and urlparts.username=="anonymous"):
7671 sftp_password = "anonymous";
7672 else:
7673 sftp_password = "";
7674 if(urlparts.scheme=="ftp"):
7675 return upload_file_to_ftp_file(url);
7676 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7677 return False;
7678 if(urlparts.scheme!="sftp"):
7679 return False;
7680 try:
7681 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7682 except paramiko.ssh_exception.SSHException:
7683 return False;
7684 except socket.gaierror:
7685 log.info("Error With URL "+url);
7686 return False;
7687 except socket.timeout:
7688 log.info("Error With URL "+url);
7689 return False;
7690 sftp = ssh.open_sftp();
7691 sftp.putfo(sftpfile, urlparts.path);
7692 sftp.close();
7693 ssh.close();
7694 sftpfile.seek(0, 0);
7695 return sftpfile;
7696 else:
7697 def upload_file_to_pysftp_file(sftpfile, url):
7698 return False;
7699
7700 if(havepysftp):
7701 def upload_file_to_pysftp_string(sftpstring, url):
7702 sftpfileo = BytesIO(sftpstring);
7703 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7704 sftpfileo.close();
7705 return sftpfile;
7706 else:
7707 def upload_file_to_pysftp_string(url):
7708 return False;
7709
7710 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7711 urlparts = urlparse(url);
7712 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7713 return download_file_from_http_file(url, headers);
7714 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7715 return download_file_from_ftp_file(url);
7716 elif(urlparts.scheme=="sftp"):
7717 if(__use_pysftp__ and havepysftp):
7718 return download_file_from_pysftp_file(url);
7719 else:
7720 return download_file_from_sftp_file(url);
7721 else:
7722 return False;
7723 return False;
7724
7725 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7726 formatspecs = FormatSpecsListToDict(formatspecs);
7727 fp = download_file_from_internet_file(url);
7728 fp = UncompressArchiveFile(fp, formatspecs);
7729 fp.seek(0, 0);
7730 if(not fp):
7731 return False;
7732 return fp;
7733
7734 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7735 urlparts = urlparse(url);
7736 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7737 return download_file_from_http_string(url, headers);
7738 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7739 return download_file_from_ftp_string(url);
7740 elif(urlparts.scheme=="sftp"):
7741 if(__use_pysftp__ and havepysftp):
7742 return download_file_from_pysftp_string(url);
7743 else:
7744 return download_file_from_sftp_string(url);
7745 else:
7746 return False;
7747 return False;
7748
7749 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7750 formatspecs = FormatSpecsListToDict(formatspecs);
7751 fp = download_file_from_internet_string(url);
7752 fp = UncompressArchiveFile(fp, formatspecs);
7753 fp.seek(0, 0);
7754 if(not fp):
7755 return False;
7756 return fp;
7757
7758 def upload_file_to_internet_file(ifp, url):
7759 urlparts = urlparse(url);
7760 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7761 return False;
7762 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7763 return upload_file_to_ftp_file(ifp, url);
7764 elif(urlparts.scheme=="sftp"):
7765 if(__use_pysftp__ and havepysftp):
7766 return upload_file_to_pysftp_file(ifp, url);
7767 else:
7768 return upload_file_to_sftp_file(ifp, url);
7769 else:
7770 return False;
7771 return False;
7772
7773 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7774 formatspecs = FormatSpecsListToDict(formatspecs);
7775 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7776 if(not catfileout):
7777 return False;
7778 catfp.seek(0, 0);
7779 upload_file_to_internet_file(catfp, outfile);
7780 return True;
7781
7782 def upload_file_to_internet_string(ifp, url):
7783 urlparts = urlparse(url);
7784 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7785 return False;
7786 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7787 return upload_file_to_ftp_string(ifp, url);
7788 elif(urlparts.scheme=="sftp"):
7789 if(__use_pysftp__ and havepysftp):
7790 return upload_file_to_pysftp_string(ifp, url);
7791 else:
7792 return upload_file_to_sftp_string(ifp, url);
7793 else:
7794 return False;
7795 return False;
7796
7797 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7798 formatspecs = FormatSpecsListToDict(formatspecs);
7799 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7800 if(not catfileout):
7801 return False;
7802 catfp.seek(0, 0);
7803 upload_file_to_internet_file(catfp, outfile);
7804 return True;
7805
7806 try:
7807 if(hasattr(shutil, "register_archive_format")):
7808 # Register the packing format
7809 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7810 except shutil.RegistryError:
7811 pass;
7812
7813 try:
7814 if(hasattr(shutil, "register_unpack_format")):
7815 # Register the unpacking format
7816 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7817 except shutil.RegistryError:
7818 pass;
A tar like file format name catfile after unix cat command (concatenate files).