Add files via upload
[PyCatFile.git] / pycatfile.py
blobf4f04540a5e2ba264024de53267a98edf2441acf
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/31/2024 Ver. 0.12.0 RC 1 - Author: cooldude2k $
18 '''
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
39 # URL Parsing
40 try:
41 from urllib.parse import urlparse, urlunparse;
42 except ImportError:
43 from urlparse import urlparse, urlunparse;
45 # Windows-specific setup
46 if os.name == 'nt':
47 if sys.version_info[0] == 2:
48 import codecs;
49 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
50 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
51 else:
52 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
53 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
55 hashlib_guaranteed = False;
56 # Environment setup
57 os.environ["PYTHONIOENCODING"] = "UTF-8";
58 os.environ["LC_CTYPE"] = "UTF-8";
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys.version_info[0] == 2:
62 try:
63 reload(sys);
64 sys.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
66 pass;
68 # CRC32 import
69 try:
70 from zlib import crc32;
71 except ImportError:
72 from binascii import crc32;
74 # Define FileNotFoundError for Python 2
75 try:
76 FileNotFoundError;
77 except NameError:
78 FileNotFoundError = IOError;
80 # RAR file support
81 rarfile_support = False;
82 try:
83 import rarfile;
84 rarfile_support = True;
85 except ImportError:
86 pass;
88 # 7z file support
89 py7zr_support = False;
90 try:
91 import py7zr;
92 py7zr_support = True;
93 except ImportError:
94 pass;
96 # TAR file checking
97 try:
98 from xtarfile import is_tarfile;
99 except ImportError:
100 try:
101 from safetar import is_tarfile;
102 except ImportError:
103 from tarfile import is_tarfile;
105 # TAR file module
106 try:
107 import xtarfile as tarfile;
108 except ImportError:
109 try:
110 import safetar as tarfile;
111 except ImportError:
112 import tarfile;
114 # Paramiko support
115 haveparamiko = False;
116 try:
117 import paramiko;
118 haveparamiko = True;
119 except ImportError:
120 pass;
122 # PySFTP support
123 havepysftp = False;
124 try:
125 import pysftp;
126 havepysftp = True;
127 except ImportError:
128 pass;
130 # Requests support
131 haverequests = False;
132 try:
133 import requests;
134 haverequests = True;
135 import urllib3;
136 logging.getLogger("urllib3").setLevel(logging.WARNING);
137 except ImportError:
138 pass;
140 # HTTP and URL parsing
141 try:
142 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
143 from urllib.parse import urlparse;
144 except ImportError:
145 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
146 from urlparse import urlparse;
148 # StringIO and BytesIO
149 try:
150 from io import StringIO, BytesIO;
151 except ImportError:
152 try:
153 from cStringIO import StringIO;
154 from cStringIO import StringIO as BytesIO;
155 except ImportError:
156 from StringIO import StringIO;
157 from StringIO import StringIO as BytesIO;
159 __use_pysftp__ = False;
160 if(not havepysftp):
161 __use_pysftp__ = False;
162 __file_format_name__ = "CatFile";
163 __program_name__ = "Py"+__file_format_name__;
164 __file_format_lower__ = __file_format_name__.lower();
165 __file_format_magic__ = __file_format_name__;
166 __file_format_len__ = len(__file_format_magic__);
167 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__ = "\x00";
169 __file_format_ver__ = "001";
170 __use_new_style__ = True;
171 __use_advanced_list__ = True;
172 __use_alt_inode__ = False;
173 __file_format_extension__ = ".cat";
174 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
175 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
176 __project__ = __program_name__;
177 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
178 __version_info__ = (0, 12, 0, "RC 1", 1);
179 __version_date_info__ = (2024, 5, 31, "RC 1", 1);
180 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
181 __revision__ = __version_info__[3];
182 __revision_id__ = "$Id$";
183 if(__version_info__[4] is not None):
184 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
185 if(__version_info__[4] is None):
186 __version_date_plusrc__ = __version_date__;
187 if(__version_info__[3] is not None):
188 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
189 if(__version_info__[3] is None):
190 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
192 PyBitness = platform.architecture();
193 if(PyBitness=="32bit" or PyBitness=="32"):
194 PyBitness = "32";
195 elif(PyBitness=="64bit" or PyBitness=="64"):
196 PyBitness = "64";
197 else:
198 PyBitness = "32";
200 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
201 if(platform.python_implementation()!=""):
202 py_implementation = platform.python_implementation();
203 if(platform.python_implementation()==""):
204 py_implementation = "Python";
205 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
206 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
207 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
208 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
209 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
210 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 def CompressionSupport():
214 compression_list = [];
215 try:
216 import gzip;
217 compression_list.append("gz");
218 compression_list.append("gzip");
219 except ImportError:
220 '''return False;'''
221 try:
222 import bz2;
223 compression_list.append("bz2");
224 compression_list.append("bzip2");
225 except ImportError:
226 '''return False;'''
227 try:
228 import lz4;
229 compression_list.append("lz4");
230 except ImportError:
231 '''return False;'''
232 try:
233 import lzo;
234 compression_list.append("lzo");
235 compression_list.append("lzop");
236 except ImportError:
237 '''return False;'''
238 try:
239 import zstandard;
240 compression_list.append("zstd");
241 compression_list.append("zstandard");
242 except ImportError:
243 '''return False;'''
244 try:
245 import lzma;
246 compression_list.append("lzma");
247 compression_list.append("xz");
248 except ImportError:
249 '''return False;'''
250 try:
251 from backports import lzma;
252 compression_list.append("lzma");
253 compression_list.append("xz");
254 except ImportError:
255 '''return False;'''
256 return compression_list;
258 compressionsupport = CompressionSupport();
259 compressionlist = ['auto'];
260 compressionlistalt = [];
261 outextlist = [];
262 outextlistwd = [];
263 if('gzip' in compressionsupport):
264 compressionlist.append('gzip');
265 compressionlistalt.append('gzip');
266 outextlist.append('gz');
267 outextlistwd.append('.gz');
268 if('bzip2' in compressionsupport):
269 compressionlist.append('bzip2');
270 compressionlistalt.append('bzip2');
271 outextlist.append('bz2');
272 outextlistwd.append('.bz2');
273 if('zstd' in compressionsupport):
274 compressionlist.append('zstd');
275 compressionlistalt.append('zstd');
276 outextlist.append('zst');
277 outextlistwd.append('.zst');
278 if('lz4' in compressionsupport):
279 compressionlist.append('lz4');
280 compressionlistalt.append('lz4');
281 outextlist.append('lz4');
282 outextlistwd.append('.lz4');
283 if('lzo' in compressionsupport):
284 compressionlist.append('lzo');
285 compressionlistalt.append('lzo');
286 outextlist.append('lzo');
287 outextlistwd.append('.lzo');
288 if('lzop' in compressionsupport):
289 compressionlist.append('lzop');
290 compressionlistalt.append('lzop');
291 outextlist.append('lzop');
292 outextlistwd.append('.lzop');
293 if('lzma' in compressionsupport):
294 compressionlist.append('lzma');
295 compressionlistalt.append('lzma');
296 outextlist.append('lzma');
297 outextlistwd.append('.lzma');
298 if('xz' in compressionsupport):
299 compressionlist.append('xz');
300 compressionlistalt.append('xz');
301 outextlist.append('xz');
302 outextlistwd.append('.xz');
304 tarfile_mimetype = "application/tar";
305 tarfile_tar_mimetype = tarfile_mimetype;
306 zipfile_mimetype = "application/zip";
307 zipfile_zip_mimetype = zipfile_mimetype;
308 rarfile_mimetype = "application/rar";
309 rarfile_rar_mimetype = rarfile_mimetype;
310 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
311 archivefile_cat_mimetype = archivefile_mimetype;
312 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
313 archivefile_gz_mimetype = archivefile_gzip_mimetype;
314 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
315 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
316 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
317 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
318 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
319 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
320 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
321 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
322 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
323 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
325 if __name__ == "__main__":
326 import subprocess;
327 curscrpath = os.path.dirname(sys.argv[0]);
328 if(curscrpath==""):
329 curscrpath = ".";
330 if(os.sep=="\\"):
331 curscrpath = curscrpath.replace(os.sep, "/");
332 curscrpath = curscrpath + "/";
333 scrfile = curscrpath + "catfile.py";
334 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
335 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
336 scrcmd.wait();
338 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
339 if(not dbgenable):
340 return True;
341 log_functions = {
342 "print": print,
343 "log": logging.info,
344 "warning": logging.warning,
345 "error": logging.error,
346 "critical": logging.critical,
347 "exception": logging.exception,
348 "logalt": lambda x: logging.log(dgblevel, x),
349 "debug": logging.debug
351 log_function = log_functions.get(outtype);
352 if(log_function):
353 log_function(dbgtxt);
354 return True;
355 return False;
357 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
358 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
359 return dbgtxt;
361 def RemoveWindowsPath(dpath):
362 if(dpath is None):
363 dpath = "";
364 if(os.sep!="/"):
365 dpath = dpath.replace(os.path.sep, "/");
366 dpath = dpath.rstrip("/");
367 if(dpath=="." or dpath==".."):
368 dpath = dpath + "/";
369 return dpath;
371 def NormalizeRelativePath(inpath):
372 inpath = RemoveWindowsPath(inpath);
373 if(os.path.isabs(inpath)):
374 outpath = inpath;
375 else:
376 if(inpath.startswith("./") or inpath.startswith("../")):
377 outpath = inpath;
378 else:
379 outpath = "./" + inpath;
380 return outpath;
382 def PrependPath(base_dir, child_path):
383 # Check if base_dir is None or empty, if so, return child_path as is
384 if not base_dir:
385 return child_path;
386 # Ensure base_dir ends with exactly one slash
387 if not base_dir.endswith('/'):
388 base_dir += '/';
389 # Check if child_path starts with ./ or ../ (indicating a relative path)
390 if child_path.startswith('./') or child_path.startswith('../'):
391 # For relative paths, we don't alter the child_path
392 return base_dir + child_path;
393 else:
394 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
395 return base_dir + child_path.lstrip('/');
397 def ListDir(dirpath, followlink=False, duplicates=False):
398 if isinstance(dirpath, (list, tuple, )):
399 dirpath = list(filter(None, dirpath));
400 elif isinstance(dirpath, str):
401 dirpath = list(filter(None, [dirpath]));
402 retlist = [];
403 fs_encoding = sys.getfilesystemencoding();
404 for mydirfile in dirpath:
405 if not os.path.exists(mydirfile):
406 return False;
407 mydirfile = NormalizeRelativePath(mydirfile);
408 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
409 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
410 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
411 for root, dirs, filenames in os.walk(mydirfile):
412 dpath = root;
413 dpath = RemoveWindowsPath(dpath);
414 if fs_encoding != 'utf-8':
415 dpath = dpath.encode(fs_encoding).decode('utf-8');
416 if dpath not in retlist and not duplicates:
417 retlist.append(dpath);
418 if duplicates:
419 retlist.append(dpath);
420 for file in filenames:
421 fpath = os.path.join(root, file);
422 fpath = RemoveWindowsPath(fpath);
423 if fs_encoding != 'utf-8':
424 fpath = fpath.encode(fs_encoding).decode('utf-8');
425 if fpath not in retlist and not duplicates:
426 retlist.append(fpath);
427 if duplicates:
428 retlist.append(fpath);
429 else:
430 path = RemoveWindowsPath(mydirfile);
431 if fs_encoding != 'utf-8':
432 path = path.encode(fs_encoding).decode('utf-8');
433 retlist.append(path);
434 return retlist;
436 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
437 if isinstance(dirpath, (list, tuple, )):
438 dirpath = list(filter(None, dirpath));
439 elif isinstance(dirpath, str):
440 dirpath = list(filter(None, [dirpath]));
441 retlist = [];
442 fs_encoding = sys.getfilesystemencoding();
443 for mydirfile in dirpath:
444 if not os.path.exists(mydirfile):
445 return False;
446 mydirfile = NormalizeRelativePath(mydirfile);
447 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
448 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
449 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
450 for root, dirs, filenames in os.walk(mydirfile):
451 # Sort dirs and filenames alphabetically in place
452 dirs.sort(key=lambda x: x.lower());
453 filenames.sort(key=lambda x: x.lower());
454 dpath = RemoveWindowsPath(root);
455 if fs_encoding != 'utf-8':
456 dpath = dpath.encode(fs_encoding).decode('utf-8');
457 if not duplicates and dpath not in retlist:
458 retlist.append(dpath);
459 elif duplicates:
460 retlist.append(dpath);
461 for file in filenames:
462 fpath = os.path.join(root, file);
463 fpath = RemoveWindowsPath(fpath);
464 if fs_encoding != 'utf-8':
465 fpath = fpath.encode(fs_encoding).decode('utf-8');
466 if not duplicates and fpath not in retlist:
467 retlist.append(fpath);
468 elif duplicates:
469 retlist.append(fpath);
470 else:
471 path = RemoveWindowsPath(mydirfile);
472 if fs_encoding != 'utf-8':
473 path = path.encode(fs_encoding).decode('utf-8');
474 retlist.append(path);
475 return retlist;
477 def create_alias_function(prefix, base_name, suffix, target_function):
478 # Define a new function that wraps the target function
479 def alias_function(*args, **kwargs):
480 return target_function(*args, **kwargs);
481 # Create the function name by combining the prefix, base name, and the suffix
482 function_name = "{}{}{}".format(prefix, base_name, suffix);
483 # Add the new function to the global namespace
484 globals()[function_name] = alias_function;
486 def create_alias_function_alt(prefix, base_name, suffix, target_function):
487 # Create the function name by combining the prefix, base name, and the suffix
488 # Use the format method for string formatting, compatible with Python 2 and 3
489 function_name = "{}{}{}".format(prefix, base_name, suffix)
490 # Add the new function (alias of the target_function) to the global namespace
491 # This line is compatible as-is with both Python 2 and 3
492 globals()[function_name] = target_function
494 def FormatSpecsListToDict(formatspecs=__file_format_list__):
495 if(isinstance(formatspecs, (list, tuple, ))):
496 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
497 elif(isinstance(formatspecs, (dict, ))):
498 return formatspecs;
499 else:
500 return __file_format_dict__;
501 return __file_format_dict__;
503 def TarFileCheck(infile):
504 try:
505 if is_tarfile(infile):
506 return True;
507 else:
508 return False;
509 except TypeError:
510 try:
511 # Check if the input is a file object
512 if hasattr(infile, 'read'):
513 # Save the current file position
514 current_position = infile.tell();
515 # Attempt to open the file object as a tar file
516 tar = tarfile.open(fileobj=infile);
517 tar.close();
518 # Restore the file position
519 infile.seek(current_position);
520 return True;
521 else:
522 # Assume it's a filename
523 tar = tarfile.open(name=infile);
524 tar.close();
525 return True;
526 except tarfile.TarError:
527 return False;
529 # initial_value can be 0xFFFF or 0x0000
530 def crc16_ansi(msg, initial_value=0xFFFF):
531 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
532 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
533 crc = initial_value; # Initial value
534 for b in msg:
535 crc ^= b << 8; # XOR byte into CRC top byte
536 for _ in range(8): # Process each bit
537 if crc & 0x8000: # If the top bit is set
538 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
539 else:
540 crc = crc << 1; # Just shift left
541 crc &= 0xFFFF; # Ensure CRC remains 16-bit
542 return crc;
544 # initial_value can be 0xFFFF or 0x0000
545 def crc16_ibm(msg, initial_value=0xFFFF):
546 return crc16_ansi(msg, initial_value);
548 # initial_value is 0xFFFF
549 def crc16(msg):
550 return crc16_ansi(msg, 0xFFFF);
552 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
553 def crc16_ccitt(msg, initial_value=0xFFFF):
554 # CRC-16-CCITT polynomial
555 poly = 0x1021; # Polynomial for CRC-16-CCITT
556 # Use the specified initial value
557 crc = initial_value;
558 for b in msg:
559 crc ^= b << 8; # XOR byte into CRC top byte
560 for _ in range(8): # Process each bit
561 if crc & 0x8000: # If the top bit is set
562 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
563 else:
564 crc = crc << 1; # Just shift left
565 crc &= 0xFFFF; # Ensure CRC remains 16-bit
566 return crc;
568 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
569 def crc64_ecma(msg, initial_value=0x0000000000000000):
570 # CRC-64-ECMA polynomial and initial value
571 poly = 0x42F0E1EBA9EA3693;
572 crc = initial_value; # Initial value for CRC-64-ECMA
573 for b in msg:
574 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
575 for _ in range(8): # Process each bit
576 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
577 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
578 else:
579 crc <<= 1; # Just shift left if the MSB is 0
580 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
581 return crc;
583 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
584 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
585 # CRC-64-ISO polynomial and initial value
586 poly = 0x000000000000001B;
587 crc = initial_value; # Common initial value for CRC-64-ISO
588 for b in msg:
589 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
590 for _ in range(8): # Process each bit
591 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
592 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
593 else:
594 crc <<= 1; # Just shift left if the MSB is 0
595 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
596 return crc;
598 def GetDataFromArray(data, path, default=None):
599 element = data;
600 try:
601 for key in path:
602 element = element[key];
603 return element;
604 except (KeyError, TypeError, IndexError):
605 return default;
607 def GetDataFromArrayAlt(structure, path, default=None):
608 element = structure;
609 for key in path:
610 if isinstance(element, dict) and key in element:
611 element = element[key];
612 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
613 element = element[key];
614 else:
615 return default;
616 return element;
618 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
619 formatspecs = FormatSpecsListToDict(formatspecs);
620 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
621 if encodedata:
622 fileheader = fileheader.encode('UTF-8');
623 checksum_methods = {
624 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
625 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
626 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
627 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
628 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
629 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
630 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
631 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
632 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
634 if checksumtype in checksum_methods:
635 return checksum_methods[checksumtype](fileheader);
636 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
637 checksumoutstr = hashlib.new(checksumtype);
638 checksumoutstr.update(fileheader);
639 return checksumoutstr.hexdigest().lower();
640 return format(0, 'x').lower();
642 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
643 formatspecs = FormatSpecsListToDict(formatspecs);
644 if encodedata:
645 instr = instr.encode('UTF-8');
646 checksum_methods = {
647 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
648 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
649 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
650 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
651 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
652 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
653 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
654 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
655 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
657 if checksumtype in checksum_methods:
658 return checksum_methods[checksumtype](instr);
659 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
660 checksumoutstr = hashlib.new(checksumtype);
661 checksumoutstr.update(instr);
662 return checksumoutstr.hexdigest().lower();
663 return format(0, 'x').lower();
665 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
666 formatspecs = FormatSpecsListToDict(formatspecs);
667 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
668 return inchecksum.lower() == catfileheadercshex;
670 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
671 formatspecs = FormatSpecsListToDict(formatspecs);
672 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
673 return inchecksum.lower() == catinfilecshex;
675 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
676 curbyte = b"";
677 curfullbyte = b"";
678 nullbyte = delimiter.encode("UTF-8");
679 while(True):
680 curbyte = fp.read(1);
681 if(curbyte==nullbyte or not curbyte):
682 break;
683 curfullbyte = curfullbyte + curbyte;
684 return curfullbyte.decode('UTF-8');
686 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
687 return ReadTillNullByteOld(fp, delimiter);
689 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
690 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
691 buffer = bytearray();
692 total_read = 0;
693 delimiter_length = len(delimiter);
694 while True:
695 chunk = fp.read(chunk_size)
696 if not chunk:
697 # End of file reached without finding the delimiter
698 break;
699 buffer.extend(chunk);
700 total_read += len(chunk);
701 if delimiter in buffer:
702 # Delimiter found, calculate where to reset the file pointer
703 index = buffer.find(delimiter);
704 # Calculate how many extra bytes were read after the delimiter
705 extra_bytes_read = len(buffer) - (index + delimiter_length);
706 # Move the file pointer back to just after the delimiter
707 fp.seek(-extra_bytes_read, 1);
708 buffer = buffer[:index];
709 break;
710 if total_read >= max_read:
711 # Stop reading if max limit is reached to prevent excessive memory usage
712 raise MemoryError("Maximum read limit reached without finding the delimiter.");
713 # Check for incomplete UTF-8 sequences at the end of the buffer
714 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
715 # This suggests that the last byte might be the start of a multi-byte character
716 # Try to read one more byte to complete the character
717 extra_byte = fp.read(1);
718 if extra_byte:
719 buffer.extend(extra_byte);
720 else:
721 # No more data available
722 break;
723 try:
724 return buffer.decode('UTF-8', errors='replace');
725 except UnicodeDecodeError:
726 return buffer.decode('UTF-8', errors='replace');
728 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
729 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
731 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
732 curfullbyte = bytearray();
733 nullbyte = delimiter.encode("UTF-8");
734 total_read = 0; # Track the total number of bytes read
735 while True:
736 curbyte = fp.read(1);
737 if curbyte == nullbyte or not curbyte:
738 break;
739 curfullbyte.extend(curbyte);
740 total_read += 1;
741 if total_read >= max_read:
742 raise MemoryError("Maximum read limit reached without finding the delimiter.");
743 # Decode the full byte array to string once out of the loop
744 try:
745 return curfullbyte.decode('UTF-8');
746 except UnicodeDecodeError:
747 # Handle potential partial UTF-8 characters
748 for i in range(1, 4):
749 try:
750 return curfullbyte[:-i].decode('UTF-8');
751 except UnicodeDecodeError:
752 continue;
753 raise; # Re-raise if decoding fails even after trimming
755 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
756 return ReadTillNullByte(fp, delimiter, max_read);
758 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
759 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
760 buffer = bytearray();
761 total_read = 0;
762 delimiter_length = len(delimiter);
763 results = [];
764 while len(results) < num_delimiters:
765 chunk = fp.read(chunk_size);
766 if not chunk:
767 # End of file reached; decode whatever is collected if it's the last needed part
768 if len(buffer) > 0:
769 results.append(buffer.decode('UTF-8', errors='replace'));
770 break;
771 buffer.extend(chunk)
772 total_read += len(chunk);
773 # Check if we have found the delimiter
774 while delimiter in buffer:
775 index = buffer.find(delimiter);
776 # Decode the section before the delimiter
777 results.append(buffer[:index].decode('UTF-8', errors='replace'));
778 # Remove the processed part from the buffer
779 buffer = buffer[index + delimiter_length:];
780 if len(results) == num_delimiters:
781 # If reached the required number of delimiters, adjust the file pointer and stop
782 fp.seek(-len(buffer), 1);
783 return results;
784 if total_read >= max_read:
785 # Stop reading if max limit is reached to prevent excessive memory usage
786 raise MemoryError("Maximum read limit reached without finding the delimiter.");
787 # Check for incomplete UTF-8 sequences at the end of the buffer
788 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
789 # This suggests that the last byte might be the start of a multi-byte character
790 # Try to read one more byte to complete the character
791 extra_byte = fp.read(1);
792 if extra_byte:
793 buffer.extend(extra_byte);
794 else:
795 # No more data available
796 break;
797 # Process remaining buffer if less than the required number of delimiters were found
798 if len(buffer) > 0 and len(results) < num_delimiters:
799 results.append(buffer.decode('UTF-8', errors='replace'));
800 return results;
802 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
803 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
805 def SeekToEndOfFile(fp):
806 lasttell = 0;
807 while(True):
808 fp.seek(1, 1);
809 if(lasttell==fp.tell()):
810 break;
811 lasttell = fp.tell();
812 return True;
814 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
815 rocount = 0;
816 roend = int(rounds);
817 HeaderOut = [];
818 while(rocount<roend):
819 HeaderOut.append(ReadTillNullByte(fp, delimiter));
820 rocount = rocount + 1;
821 return HeaderOut;
823 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
824 headerpresize = ReadTillNullByte(fp, delimiter);
825 headersize = int(headerpresize, 16);
826 if(headersize<=0):
827 return [];
828 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
829 fp.seek(1, 1);
830 rocount = 0;
831 roend = int(len(headercontent));
832 HeaderOut = [headerpresize];
833 while(rocount<roend):
834 HeaderOut.append(headercontent[rocount]);
835 rocount = rocount + 1;
836 return HeaderOut;
838 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
839 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
840 headersize = int(preheaderdata[0], 16);
841 headernumfields = int(preheaderdata[1], 16);
842 if(headersize<=0 or headernumfields<=0):
843 return [];
844 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
845 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
846 HeaderOut = preheaderdata + headerdata;
847 return HeaderOut;
849 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
850 formatspecs = FormatSpecsListToDict(formatspecs);
851 delimiter = formatspecs['format_delimiter'];
852 fheaderstart = fp.tell();
853 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
854 if(len(HeaderOut)==0):
855 return False;
856 if(re.findall(r"^[.|/]", HeaderOut[3])):
857 fname = HeaderOut[3];
858 else:
859 fname = "./"+HeaderOut[3];
860 fcs = HeaderOut[-2].lower();
861 fccs = HeaderOut[-1].lower();
862 fsize = int(HeaderOut[5], 16);
863 fcompression = HeaderOut[12];
864 fcsize = int(HeaderOut[13], 16);
865 fseeknextfile = HeaderOut[25];
866 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
867 if(fcs!=newfcs and not skipchecksum):
868 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
869 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
870 return False;
871 fhend = fp.tell() - 1;
872 fcontentstart = fp.tell();
873 fcontents = BytesIO();
874 if(fsize>0 and not listonly):
875 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
876 fcontents.write(fp.read(fsize));
877 else:
878 fcontents.write(fp.read(fcsize));
879 elif(fsize>0 and listonly):
880 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
881 fp.seek(fsize, 1);
882 else:
883 fp.seek(fcsize, 1);
884 fcontents.seek(0, 0);
885 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
886 if(fccs!=newfccs and not skipchecksum and not listonly):
887 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
888 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
889 return False;
890 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
891 pass;
892 else:
893 fcontents.seek(0, 0);
894 if(uncompress):
895 fcontents = UncompressArchiveFile(fcontents, formatspecs);
896 fcontentend = fp.tell();
897 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
898 fseeknextasnum = int(fseeknextfile.replace("+", ""));
899 if(abs(fseeknextasnum)==0):
900 pass;
901 fp.seek(fseeknextasnum, 1);
902 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
903 fseeknextasnum = int(fseeknextfile);
904 if(abs(fseeknextasnum)==0):
905 pass;
906 fp.seek(fseeknextasnum, 1);
907 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
908 fseeknextasnum = int(fseeknextfile);
909 if(abs(fseeknextasnum)==0):
910 pass;
911 fp.seek(fseeknextasnum, 0);
912 else:
913 return False;
914 HeaderOut.append(fcontents);
915 return HeaderOut;
917 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
918 formatspecs = FormatSpecsListToDict(formatspecs);
919 delimiter = formatspecs['format_delimiter'];
920 fheaderstart = fp.tell();
921 if(formatspecs['new_style']):
922 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
923 else:
924 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
925 if(len(HeaderOut)==0):
926 return False;
927 fheadsize = int(HeaderOut[0], 16);
928 fnumfields = int(HeaderOut[1], 16);
929 ftype = int(HeaderOut[2], 16);
930 if(re.findall(r"^[.|/]", HeaderOut[3])):
931 fname = HeaderOut[3];
932 else:
933 fname = "./"+HeaderOut[3];
934 fbasedir = os.path.dirname(fname);
935 flinkname = HeaderOut[4];
936 fsize = int(HeaderOut[5], 16);
937 fatime = int(HeaderOut[6], 16);
938 fmtime = int(HeaderOut[7], 16);
939 fctime = int(HeaderOut[8], 16);
940 fbtime = int(HeaderOut[9], 16);
941 fmode = int(HeaderOut[10], 16);
942 fchmode = stat.S_IMODE(fmode);
943 ftypemod = stat.S_IFMT(fmode);
944 fwinattributes = int(HeaderOut[11], 16);
945 fcompression = HeaderOut[12];
946 fcsize = int(HeaderOut[13], 16);
947 fuid = int(HeaderOut[14], 16);
948 funame = HeaderOut[15];
949 fgid = int(HeaderOut[16], 16);
950 fgname = HeaderOut[17];
951 fid = int(HeaderOut[18], 16);
952 finode = int(HeaderOut[19], 16);
953 flinkcount = int(HeaderOut[20], 16);
954 fdev_minor = int(HeaderOut[21], 16);
955 fdev_major = int(HeaderOut[22], 16);
956 frdev_minor = int(HeaderOut[23], 16);
957 frdev_major = int(HeaderOut[24], 16);
958 fseeknextfile = HeaderOut[25];
959 fextrasize = int(HeaderOut[26], 16);
960 fextrafields = int(HeaderOut[27], 16);
961 extrafieldslist = [];
962 extrastart = 28
963 extraend = extrastart + fextrafields;
964 extrafieldslist = [];
965 if(extrastart<extraend):
966 extrafieldslist.append(HeaderOut[extrastart]);
967 extrastart = extrastart + 1;
968 fcs = HeaderOut[extrastart + 2].lower();
969 fccs = HeaderOut[extrastart + 3].lower();
970 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
971 if(fcs!=newfcs and not skipchecksum):
972 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
973 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
974 return False;
975 fhend = fp.tell() - 1;
976 fcontentstart = fp.tell();
977 fcontents = BytesIO();
978 pyhascontents = False;
979 if(fsize>0 and not listonly):
980 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
981 fcontents.write(fp.read(fsize));
982 else:
983 fcontents.write(fp.read(fcsize));
984 pyhascontents = True;
985 elif(fsize>0 and listonly):
986 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
987 fp.seek(fsize, 1);
988 else:
989 fp.seek(fcsize, 1);
990 pyhascontents = False;
991 fcontents.seek(0, 0);
992 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
993 if(fccs!=newfccs and not skipchecksum and not listonly):
994 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
995 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
996 return False;
997 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
998 pass;
999 else:
1000 fcontents.seek(0, 0);
1001 if(uncompress):
1002 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1003 fcontents.seek(0, 0);
1004 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1005 fcontentend = fp.tell() - 1;
1006 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1007 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1008 if(abs(fseeknextasnum)==0):
1009 pass;
1010 fp.seek(fseeknextasnum, 1);
1011 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1012 fseeknextasnum = int(fseeknextfile);
1013 if(abs(fseeknextasnum)==0):
1014 pass;
1015 fp.seek(fseeknextasnum, 1);
1016 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1017 fseeknextasnum = int(fseeknextfile);
1018 if(abs(fseeknextasnum)==0):
1019 pass;
1020 fp.seek(fseeknextasnum, 0);
1021 else:
1022 return False;
1023 fcontents.seek(0, 0);
1024 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
1025 return catlist;
1027 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1028 formatspecs = FormatSpecsListToDict(formatspecs);
1029 delimiter = formatspecs['format_delimiter'];
1030 fheaderstart = fp.tell();
1031 if(formatspecs['new_style']):
1032 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1033 else:
1034 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1035 if(len(HeaderOut)==0):
1036 return False;
1037 fheadsize = int(HeaderOut[0], 16);
1038 fnumfields = int(HeaderOut[1], 16);
1039 ftype = int(HeaderOut[2], 16);
1040 if(re.findall(r"^[.|/]", HeaderOut[3])):
1041 fname = HeaderOut[3];
1042 else:
1043 fname = "./"+HeaderOut[3];
1044 fbasedir = os.path.dirname(fname);
1045 flinkname = HeaderOut[4];
1046 fsize = int(HeaderOut[5], 16);
1047 fatime = int(HeaderOut[6], 16);
1048 fmtime = int(HeaderOut[7], 16);
1049 fctime = int(HeaderOut[8], 16);
1050 fbtime = int(HeaderOut[9], 16);
1051 fmode = int(HeaderOut[10], 16);
1052 fchmode = stat.S_IMODE(fmode);
1053 ftypemod = stat.S_IFMT(fmode);
1054 fwinattributes = int(HeaderOut[11], 16);
1055 fcompression = HeaderOut[12];
1056 fcsize = int(HeaderOut[13], 16);
1057 fuid = int(HeaderOut[14], 16);
1058 funame = HeaderOut[15];
1059 fgid = int(HeaderOut[16], 16);
1060 fgname = HeaderOut[17];
1061 fid = int(HeaderOut[18], 16);
1062 finode = int(HeaderOut[19], 16);
1063 flinkcount = int(HeaderOut[20], 16);
1064 fdev_minor = int(HeaderOut[21], 16);
1065 fdev_major = int(HeaderOut[22], 16);
1066 frdev_minor = int(HeaderOut[23], 16);
1067 frdev_major = int(HeaderOut[24], 16);
1068 fseeknextfile = HeaderOut[25];
1069 fextrasize = int(HeaderOut[26], 16);
1070 fextrafields = int(HeaderOut[27], 16);
1071 extrafieldslist = [];
1072 extrastart = 28;
1073 extraend = extrastart + fextrafields;
1074 extrafieldslist = [];
1075 if(extrastart<extraend):
1076 extrafieldslist.append(HeaderOut[extrastart]);
1077 extrastart = extrastart + 1;
1078 fheaderchecksumtype = [extrastart].lower();
1079 fcontentchecksumtype = [extrastart + 1].lower();
1080 fcs = HeaderOut[extrastart + 2].lower();
1081 fccs = HeaderOut[extrastart + 3].lower();
1082 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1083 if(fcs!=newfcs and not skipchecksum):
1084 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1085 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1086 return False;
1087 fhend = fp.tell() - 1;
1088 fcontentstart = fp.tell();
1089 fcontents = BytesIO();
1090 pyhascontents = False;
1091 if(fsize>0 and not listonly):
1092 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1093 fcontents.write(fp.read(fsize));
1094 else:
1095 fcontents.write(fp.read(fcsize));
1096 pyhascontents = True;
1097 elif(fsize>0 and listonly):
1098 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1099 fp.seek(fsize, 1);
1100 else:
1101 fp.seek(fcsize, 1);
1102 pyhascontents = False;
1103 fcontents.seek(0, 0);
1104 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1105 if(fccs!=newfccs and not skipchecksum and not listonly):
1106 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1107 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1108 return False;
1109 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1110 pass;
1111 else:
1112 fcontents.seek(0, 0);
1113 if(uncompress):
1114 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1115 fcontents.seek(0, 0);
1116 fcontentend = fp.tell() - 1;
1117 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1118 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1119 if(abs(fseeknextasnum)==0):
1120 pass;
1121 fp.seek(fseeknextasnum, 1);
1122 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1123 fseeknextasnum = int(fseeknextfile);
1124 if(abs(fseeknextasnum)==0):
1125 pass;
1126 fp.seek(fseeknextasnum, 1);
1127 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1128 fseeknextasnum = int(fseeknextfile);
1129 if(abs(fseeknextasnum)==0):
1130 pass;
1131 fp.seek(fseeknextasnum, 0);
1132 else:
1133 return False;
1134 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1135 return catlist;
1137 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1138 formatspecs = FormatSpecsListToDict(formatspecs);
1139 delimiter = formatspecs['format_delimiter'];
1140 curloc = fp.tell();
1141 if(curloc>0):
1142 fp.seek(0, 0);
1143 catheader = ReadFileHeaderData(fp, 4, delimiter);
1144 if(curloc>0):
1145 fp.seek(curloc, 0);
1146 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1147 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1148 if(not headercheck and not skipchecksum):
1149 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1150 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1151 return False;
1152 fnumfiles = int(catheader[1], 16);
1153 countnum = 0;
1154 flist = [];
1155 while(countnum < fnumfiles):
1156 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1157 if(len(HeaderOut)==0):
1158 break;
1159 flist.append(HeaderOut);
1160 countnum = countnum + 1;
1161 return flist;
1163 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1164 formatspecs = FormatSpecsListToDict(formatspecs);
1165 delimiter = formatspecs['format_delimiter'];
1166 curloc = fp.tell();
1167 if(curloc>0):
1168 fp.seek(0, 0);
1169 catheader = ReadFileHeaderData(fp, 4, delimiter);
1170 if(curloc>0):
1171 fp.seek(curloc, 0);
1172 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1173 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1174 if(not headercheck and not skipchecksum):
1175 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1176 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1177 return False;
1178 catstring = catheader[0];
1179 catversion = re.findall(r"([\d]+)", catstring);
1180 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1181 fprenumfiles = catheader[1];
1182 fnumfiles = int(fprenumfiles, 16);
1183 fprechecksumtype = catheader[2];
1184 fprechecksum = catheader[3];
1185 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1186 if(seekstart<0 and seekstart>fnumfiles):
1187 seekstart = 0;
1188 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1189 seekend = fnumfiles;
1190 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1191 seekend = fnumfiles - abs(seekend);
1192 if(seekstart>0):
1193 il = 0;
1194 while(il < seekstart):
1195 prefhstart = fp.tell();
1196 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1197 if(len(preheaderdata)==0):
1198 break;
1199 prefsize = int(preheaderdata[5], 16);
1200 prefseeknextfile = preheaderdata[25];
1201 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1202 prefcs = preheaderdata[-2];
1203 if(prefcs!=prenewfcs and not skipchecksum):
1204 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1205 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1206 return False;
1207 valid_archive = False;
1208 invalid_archive = True;
1209 prefhend = fp.tell() - 1;
1210 prefcontentstart = fp.tell();
1211 prefcontents = BytesIO();
1212 pyhascontents = False;
1213 if(prefsize>0):
1214 prefcontents.write(fp.read(prefsize));
1215 prefcontents.seek(0, 0);
1216 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1217 prefccs = preheaderdata[-1];
1218 pyhascontents = True;
1219 if(prefccs!=prenewfccs and not skipchecksum):
1220 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1221 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1222 return False;
1223 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1224 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1225 if(abs(fseeknextasnum)==0):
1226 pass;
1227 fp.seek(fseeknextasnum, 1);
1228 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1229 fseeknextasnum = int(prefseeknextfile);
1230 if(abs(fseeknextasnum)==0):
1231 pass;
1232 fp.seek(fseeknextasnum, 1);
1233 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1234 fseeknextasnum = int(prefseeknextfile);
1235 if(abs(fseeknextasnum)==0):
1236 pass;
1237 fp.seek(fseeknextasnum, 0);
1238 else:
1239 return False;
1240 il = il + 1;
1241 realidnum = 0;
1242 countnum = seekstart;
1243 while(countnum < seekend):
1244 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1245 if(len(HeaderOut)==0):
1246 break;
1247 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1248 catlist['ffilelist'].append(HeaderOut);
1249 countnum = countnum + 1;
1250 realidnum = realidnum + 1;
1251 return catlist;
1253 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1254 formatspecs = FormatSpecsListToDict(formatspecs);
1255 delimiter = formatspecs['format_delimiter'];
1256 curloc = fp.tell();
1257 if(curloc>0):
1258 fp.seek(0, 0);
1259 catheader = ReadFileHeaderData(fp, 4, delimiter);
1260 if(curloc>0):
1261 fp.seek(curloc, 0);
1262 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1263 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1264 if(not headercheck and not skipchecksum):
1265 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1266 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1267 return False;
1268 catstring = catheader[0];
1269 catversion = re.findall(r"([\d]+)", catstring);
1270 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1271 fprenumfiles = catheader[1];
1272 fnumfiles = int(fprenumfiles, 16);
1273 fprechecksumtype = catheader[2];
1274 fprechecksum = catheader[3];
1275 catlist = [];
1276 if(seekstart<0 and seekstart>fnumfiles):
1277 seekstart = 0;
1278 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1279 seekend = fnumfiles;
1280 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1281 seekend = fnumfiles - abs(seekend);
1282 if(seekstart>0):
1283 il = 0;
1284 while(il < seekstart):
1285 prefhstart = fp.tell();
1286 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1287 if(len(preheaderdata)==0):
1288 break;
1289 prefsize = int(preheaderdata[5], 16);
1290 prefcompression = preheaderdata[12];
1291 prefcsize = int(preheaderdata[13], 16);
1292 prefseeknextfile = HeaderOut[25];
1293 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1294 prefcs = preheaderdata[-2];
1295 if(prefcs!=prenewfcs and not skipchecksum):
1296 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1297 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1298 return False;
1299 valid_archive = False;
1300 invalid_archive = True;
1301 prefhend = fp.tell() - 1;
1302 prefcontentstart = fp.tell();
1303 prefcontents = "";
1304 pyhascontents = False;
1305 if(prefsize>0):
1306 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1307 prefcontents = catfp.read(prefsize);
1308 else:
1309 prefcontents = catfp.read(prefcsize);
1310 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1311 prefccs = preheaderdata[-1];
1312 pyhascontents = True;
1313 if(prefccs!=prenewfccs and not skipchecksum):
1314 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1315 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1316 return False;
1317 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1318 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1319 if(abs(fseeknextasnum)==0):
1320 pass;
1321 catfp.seek(fseeknextasnum, 1);
1322 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1323 fseeknextasnum = int(prefseeknextfile);
1324 if(abs(fseeknextasnum)==0):
1325 pass;
1326 catfp.seek(fseeknextasnum, 1);
1327 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1328 fseeknextasnum = int(prefseeknextfile);
1329 if(abs(fseeknextasnum)==0):
1330 pass;
1331 catfp.seek(fseeknextasnum, 0);
1332 else:
1333 return False;
1334 il = il + 1;
1335 realidnum = 0;
1336 countnum = seekstart;
1337 while(countnum < seekend):
1338 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1339 if(len(HeaderOut)==0):
1340 break;
1341 catlist.append(HeaderOut);
1342 countnum = countnum + 1;
1343 realidnum = realidnum + 1;
1344 return catlist;
1346 def ReadInFileBySizeWithContentToArray(infile, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1347 formatspecs = FormatSpecsListToDict(formatspecs);
1348 delimiter = formatspecs['format_delimiter'];
1349 if(hasattr(infile, "read") or hasattr(infile, "write")):
1350 fp = infile;
1351 fp.seek(0, 0);
1352 fp = UncompressArchiveFile(fp, formatspecs);
1353 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1354 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1355 return False;
1356 if(not fp):
1357 return False;
1358 fp.seek(0, 0);
1359 elif(infile=="-"):
1360 fp = BytesIO();
1361 if(hasattr(sys.stdin, "buffer")):
1362 shutil.copyfileobj(sys.stdin.buffer, fp);
1363 else:
1364 shutil.copyfileobj(sys.stdin, fp);
1365 fp.seek(0, 0);
1366 fp = UncompressArchiveFile(fp, formatspecs);
1367 if(not fp):
1368 return False;
1369 fp.seek(0, 0);
1370 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1371 fp = download_file_from_internet_file(infile);
1372 fp = UncompressArchiveFile(fp, formatspecs);
1373 fp.seek(0, 0);
1374 if(not fp):
1375 return False;
1376 fp.seek(0, 0);
1377 else:
1378 infile = RemoveWindowsPath(infile);
1379 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1380 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1381 return False;
1382 compresscheck = CheckCompressionType(infile, formatspecs, True);
1383 if(not compresscheck):
1384 fextname = os.path.splitext(infile)[1];
1385 if(fextname==".gz"):
1386 compresscheck = "gzip";
1387 elif(fextname==".bz2"):
1388 compresscheck = "bzip2";
1389 elif(fextname==".zst"):
1390 compresscheck = "zstd";
1391 elif(fextname==".lz4" or fextname==".clz4"):
1392 compresscheck = "lz4";
1393 elif(fextname==".lzo" or fextname==".lzop"):
1394 compresscheck = "lzo";
1395 elif(fextname==".lzma" or fextname==".xz"):
1396 compresscheck = "lzma";
1397 else:
1398 return False;
1399 if(not compresscheck):
1400 return False;
1401 fp = UncompressFile(infile, formatspecs, "rb");
1402 return ReadFileDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1404 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1405 formatspecs = FormatSpecsListToDict(formatspecs);
1406 delimiter = formatspecs['format_delimiter'];
1407 if(hasattr(infile, "read") or hasattr(infile, "write")):
1408 fp = infile;
1409 fp.seek(0, 0);
1410 fp = UncompressArchiveFile(fp, formatspecs);
1411 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1412 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1413 return False;
1414 if(not fp):
1415 return False;
1416 fp.seek(0, 0);
1417 elif(infile=="-"):
1418 fp = BytesIO();
1419 if(hasattr(sys.stdin, "buffer")):
1420 shutil.copyfileobj(sys.stdin.buffer, fp);
1421 else:
1422 shutil.copyfileobj(sys.stdin, fp);
1423 fp.seek(0, 0);
1424 fp = UncompressArchiveFile(fp, formatspecs);
1425 if(not fp):
1426 return False;
1427 fp.seek(0, 0);
1428 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1429 fp = download_file_from_internet_file(infile);
1430 fp = UncompressArchiveFile(fp, formatspecs);
1431 fp.seek(0, 0);
1432 if(not fp):
1433 return False;
1434 fp.seek(0, 0);
1435 else:
1436 infile = RemoveWindowsPath(infile);
1437 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1438 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1439 return False;
1440 compresscheck = CheckCompressionType(infile, formatspecs, True);
1441 if(not compresscheck):
1442 fextname = os.path.splitext(infile)[1];
1443 if(fextname==".gz"):
1444 compresscheck = "gzip";
1445 elif(fextname==".bz2"):
1446 compresscheck = "bzip2";
1447 elif(fextname==".zst"):
1448 compresscheck = "zstd";
1449 elif(fextname==".lz4" or fextname==".clz4"):
1450 compresscheck = "lz4";
1451 elif(fextname==".lzo" or fextname==".lzop"):
1452 compresscheck = "lzo";
1453 elif(fextname==".lzma" or fextname==".xz"):
1454 compresscheck = "lzma";
1455 else:
1456 return False;
1457 if(not compresscheck):
1458 return False;
1459 fp = UncompressFile(infile, formatspecs, "rb");
1460 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1462 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1463 outdata = str(indata) + delimiter;
1464 return outdata;
1466 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1467 outdata = "";
1468 inum = 0;
1469 il = len(indata);
1470 while(inum < il):
1471 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1472 inum = inum + 1;
1473 return outdata;
1475 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1476 formatspecs = FormatSpecsListToDict(formatspecs);
1477 delimiter = formatspecs['format_delimiter'];
1478 catver = formatspecs['format_ver'];
1479 fileheaderver = str(int(catver.replace(".", "")));
1480 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1481 fp.write(fileheader.encode('UTF-8'));
1482 fnumfiles = format(int(numfiles), 'x').lower();
1483 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1484 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1485 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1486 fp.write(fnumfilesa.encode('UTF-8'));
1487 try:
1488 fp.flush();
1489 os.fsync(fp.fileno());
1490 except io.UnsupportedOperation:
1491 pass;
1492 except AttributeError:
1493 pass;
1494 except OSError as e:
1495 pass;
1496 return fp;
1498 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1499 formatspecs = FormatSpecsListToDict(formatspecs);
1500 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1501 return fp;
1503 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1504 formatspecs = FormatSpecsListToDict(formatspecs);
1505 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1506 if(os.path.exists(outfile)):
1507 try:
1508 os.unlink(outfile);
1509 except OSError as e:
1510 pass;
1511 if(outfile=="-"):
1512 verbose = False;
1513 catfpfp = BytesIO();
1514 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1515 catfp = outfile;
1516 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1517 catfp = BytesIO();
1518 else:
1519 fbasename = os.path.splitext(outfile)[0];
1520 fextname = os.path.splitext(outfile)[1];
1521 catfp = CompressOpenFile(outfile, True, compressionlevel);
1522 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1523 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1524 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1525 try:
1526 catfp.flush();
1527 os.fsync(catfp.fileno());
1528 except io.UnsupportedOperation:
1529 pass;
1530 except AttributeError:
1531 pass;
1532 except OSError as e:
1533 pass;
1534 if(outfile=="-"):
1535 catfp.seek(0, 0);
1536 if(hasattr(sys.stdout, "buffer")):
1537 shutil.copyfileobj(catfp, sys.stdout.buffer);
1538 else:
1539 shutil.copyfileobj(catfp, sys.stdout);
1540 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1541 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1542 catfp.seek(0, 0);
1543 upload_file_to_internet_file(catfp, outfile);
1544 if(returnfp):
1545 catfp.seek(0, 0);
1546 return catfp;
1547 else:
1548 catfp.close();
1549 return True;
1551 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1552 formatspecs = FormatSpecsListToDict(formatspecs);
1553 extrafields = format(len(extradata), 'x').lower();
1554 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1555 if(len(extradata)>0):
1556 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1557 extrasizelen = format(len(extrasizestr), 'x').lower();
1558 catoutlen = len(filevalues) + len(extradata) + 5;
1559 catoutlenhex = format(catoutlen, 'x').lower();
1560 catoutlist = filevalues;
1561 catoutlist.insert(0, catoutlenhex);
1562 catoutlist.append(extrasizelen);
1563 catoutlist.append(extrafields);
1564 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1565 if(len(extradata)>0):
1566 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1567 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
1568 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1569 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1570 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1571 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1572 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1573 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1574 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1575 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1576 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1577 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1578 fp.write(catfileout);
1579 try:
1580 fp.flush();
1581 os.fsync(fp.fileno());
1582 except io.UnsupportedOperation:
1583 pass;
1584 except AttributeError:
1585 pass;
1586 except OSError as e:
1587 pass;
1588 return fp;
1590 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1591 formatspecs = FormatSpecsListToDict(formatspecs);
1592 advancedlist = formatspecs['use_advanced_list'];
1593 altinode = formatspecs['use_alt_inode'];
1594 if(verbose):
1595 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1596 if(infiles=="-"):
1597 for line in sys.stdin:
1598 infilelist.append(line.strip());
1599 infilelist = list(filter(None, infilelist));
1600 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1601 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1602 return False;
1603 with UncompressFile(infiles, formatspecs, "r") as finfile:
1604 for line in finfile:
1605 infilelist.append(line.strip());
1606 infilelist = list(filter(None, infilelist));
1607 else:
1608 if(isinstance(infiles, (list, tuple, ))):
1609 infilelist = list(filter(None, infiles));
1610 elif(isinstance(infiles, (str, ))):
1611 infilelist = list(filter(None, [infiles]));
1612 if(advancedlist):
1613 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1614 else:
1615 GetDirList = ListDir(infilelist, followlink, False);
1616 if(not GetDirList):
1617 return False;
1618 curinode = 0;
1619 curfid = 0;
1620 inodelist = [];
1621 inodetofile = {};
1622 filetoinode = {};
1623 inodetocatinode = {};
1624 numfiles = int(len(GetDirList));
1625 fnumfiles = format(numfiles, 'x').lower();
1626 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1627 for curfname in GetDirList:
1628 if(re.findall(r"^[.|/]", curfname)):
1629 fname = curfname;
1630 else:
1631 fname = "./"+curfname;
1632 if(verbose):
1633 VerbosePrintOut(fname);
1634 if(not followlink or followlink is None):
1635 fstatinfo = os.lstat(fname);
1636 else:
1637 fstatinfo = os.stat(fname);
1638 fpremode = fstatinfo.st_mode;
1639 finode = fstatinfo.st_ino;
1640 flinkcount = fstatinfo.st_nlink;
1641 ftype = 0;
1642 if(stat.S_ISREG(fpremode)):
1643 ftype = 0;
1644 elif(stat.S_ISLNK(fpremode)):
1645 ftype = 2;
1646 elif(stat.S_ISCHR(fpremode)):
1647 ftype = 3;
1648 elif(stat.S_ISBLK(fpremode)):
1649 ftype = 4;
1650 elif(stat.S_ISDIR(fpremode)):
1651 ftype = 5;
1652 elif(stat.S_ISFIFO(fpremode)):
1653 ftype = 6;
1654 elif(stat.S_ISSOCK(fpremode)):
1655 ftype = 8;
1656 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1657 ftype = 9;
1658 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1659 ftype = 10;
1660 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1661 ftype = 11;
1662 else:
1663 ftype = 0;
1664 flinkname = "";
1665 fcurfid = format(int(curfid), 'x').lower();
1666 if(not followlink and finode!=0):
1667 if(ftype!=1):
1668 if(finode in inodelist):
1669 ftype = 1;
1670 flinkname = inodetofile[finode];
1671 if(altinode):
1672 fcurinode = format(int(finode), 'x').lower();
1673 else:
1674 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1675 if(finode not in inodelist):
1676 inodelist.append(finode);
1677 inodetofile.update({finode: fname});
1678 inodetocatinode.update({finode: curinode});
1679 if(altinode):
1680 fcurinode = format(int(finode), 'x').lower();
1681 else:
1682 fcurinode = format(int(curinode), 'x').lower();
1683 curinode = curinode + 1;
1684 else:
1685 fcurinode = format(int(curinode), 'x').lower();
1686 curinode = curinode + 1;
1687 curfid = curfid + 1;
1688 if(ftype==2):
1689 flinkname = os.readlink(fname);
1690 fdev = fstatinfo.st_dev;
1691 getfdev = GetDevMajorMinor(fdev);
1692 fdev_minor = getfdev[0];
1693 fdev_major = getfdev[1];
1694 frdev = fstatinfo.st_dev;
1695 if(hasattr(fstatinfo, "st_rdev")):
1696 frdev = fstatinfo.st_rdev;
1697 else:
1698 frdev = fstatinfo.st_dev;
1699 getfrdev = GetDevMajorMinor(frdev);
1700 frdev_minor = getfrdev[0];
1701 frdev_major = getfrdev[1];
1702 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1703 fsize = format(int("0"), 'x').lower();
1704 elif(ftype==0 or ftype==7):
1705 fsize = format(int(fstatinfo.st_size), 'x').lower();
1706 else:
1707 fsize = format(int(fstatinfo.st_size)).lower();
1708 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1709 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1710 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1711 if(hasattr(fstatinfo, "st_birthtime")):
1712 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1713 else:
1714 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1715 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1716 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1717 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1718 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1719 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1720 funame = "";
1721 try:
1722 import pwd;
1723 try:
1724 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1725 funame = userinfo.pw_name;
1726 except KeyError:
1727 funame = "";
1728 except ImportError:
1729 funame = "";
1730 fgname = "";
1731 try:
1732 import grp;
1733 try:
1734 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1735 fgname = groupinfo.gr_name;
1736 except KeyError:
1737 fgname = "";
1738 except ImportError:
1739 fgname = "";
1740 fdev_minor = format(int(fdev_minor), 'x').lower();
1741 fdev_major = format(int(fdev_major), 'x').lower();
1742 frdev_minor = format(int(frdev_minor), 'x').lower();
1743 frdev_major = format(int(frdev_major), 'x').lower();
1744 finode = format(int(finode), 'x').lower();
1745 flinkcount = format(int(flinkcount), 'x').lower();
1746 if(hasattr(fstatinfo, "st_file_attributes")):
1747 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1748 else:
1749 fwinattributes = format(int(0), 'x').lower();
1750 fcompression = "";
1751 fcsize = format(int(0), 'x').lower();
1752 fcontents = BytesIO();
1753 chunk_size = 1024;
1754 if(ftype==0 or ftype==7):
1755 with open(fname, "rb") as fpc:
1756 shutil.copyfileobj(fpc, fcontents);
1757 if(not compresswholefile):
1758 fcontents.seek(0, 2);
1759 ucfsize = fcontents.tell();
1760 fcontents.seek(0, 0);
1761 if(compression=="auto"):
1762 ilsize = len(compressionlistalt);
1763 ilmin = 0;
1764 ilcsize = [];
1765 while(ilmin < ilsize):
1766 cfcontents = BytesIO();
1767 shutil.copyfileobj(fcontents, cfcontents);
1768 fcontents.seek(0, 0);
1769 cfcontents.seek(0, 0);
1770 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1771 if(cfcontents):
1772 cfcontents.seek(0, 2);
1773 ilcsize.append(cfcontents.tell());
1774 cfcontents.close();
1775 else:
1776 try:
1777 ilcsize.append(sys.maxint);
1778 except AttributeError:
1779 ilcsize.append(sys.maxsize);
1780 ilmin = ilmin + 1;
1781 ilcmin = ilcsize.index(min(ilcsize));
1782 compression = compressionlistalt[ilcmin];
1783 fcontents.seek(0, 0);
1784 cfcontents = BytesIO();
1785 shutil.copyfileobj(fcontents, cfcontents);
1786 cfcontents.seek(0, 0);
1787 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1788 cfcontents.seek(0, 2);
1789 cfsize = cfcontents.tell();
1790 if(ucfsize > cfsize):
1791 fcsize = format(int(cfsize), 'x').lower();
1792 fcompression = compression;
1793 fcontents.close();
1794 fcontents = cfcontents;
1795 if(followlink and (ftype==1 or ftype==2)):
1796 flstatinfo = os.stat(flinkname);
1797 with open(flinkname, "rb") as fpc:
1798 shutil.copyfileobj(fpc, fcontents);
1799 if(not compresswholefile):
1800 fcontents.seek(0, 2);
1801 ucfsize = fcontents.tell();
1802 fcontents.seek(0, 0);
1803 if(compression=="auto"):
1804 ilsize = len(compressionlistalt);
1805 ilmin = 0;
1806 ilcsize = [];
1807 while(ilmin < ilsize):
1808 cfcontents = BytesIO();
1809 shutil.copyfileobj(fcontents, cfcontents);
1810 fcontents.seek(0, 0);
1811 cfcontents.seek(0, 0);
1812 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1813 if(cfcontents):
1814 cfcontents.seek(0, 2);
1815 ilcsize.append(cfcontents.tell());
1816 cfcontents.close();
1817 else:
1818 try:
1819 ilcsize.append(sys.maxint);
1820 except AttributeError:
1821 ilcsize.append(sys.maxsize);
1822 ilmin = ilmin + 1;
1823 ilcmin = ilcsize.index(min(ilcsize));
1824 compression = compressionlistalt[ilcmin];
1825 fcontents.seek(0, 0);
1826 cfcontents = BytesIO();
1827 shutil.copyfileobj(fcontents, cfcontents);
1828 cfcontents.seek(0, 0);
1829 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1830 cfcontents.seek(0, 2);
1831 cfsize = cfcontents.tell();
1832 if(ucfsize > cfsize):
1833 fcsize = format(int(cfsize), 'x').lower();
1834 fcompression = compression;
1835 fcontents.close();
1836 fcontents = cfcontents;
1837 if(fcompression=="none"):
1838 fcompression = "";
1839 fcontents.seek(0, 0);
1840 ftypehex = format(ftype, 'x').lower();
1841 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
1842 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1843 if(numfiles>0):
1844 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1845 fp.seek(0, 0);
1846 return fp;
1848 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1849 formatspecs = FormatSpecsListToDict(formatspecs);
1850 if(verbose):
1851 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1852 GetDirList = inlist;
1853 if(not GetDirList):
1854 return False;
1855 curinode = 0;
1856 curfid = 0;
1857 inodelist = [];
1858 inodetofile = {};
1859 filetoinode = {};
1860 inodetocatinode = {};
1861 numfiles = int(len(GetDirList));
1862 fnumfiles = format(numfiles, 'x').lower();
1863 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1864 for curfname in GetDirList:
1865 ftype = format(curfname[0], 'x').lower();
1866 if(re.findall(r"^[.|/]", curfname[1])):
1867 fname = curfname[1];
1868 else:
1869 fname = "./"+curfname[1];
1870 fbasedir = os.path.dirname(fname);
1871 flinkname = curfname[2];
1872 fsize = format(curfname[3], 'x').lower();
1873 fatime = format(curfname[4], 'x').lower();
1874 fmtime = format(curfname[5], 'x').lower();
1875 fctime = format(curfname[6], 'x').lower();
1876 fbtime = format(curfname[7], 'x').lower();
1877 fmode = format(curfname[8], 'x').lower();
1878 fwinattributes = format(curfname[9], 'x').lower();
1879 fcompression = curfname[10];
1880 fcsize = format(curfname[11], 'x').lower();
1881 fuid = format(curfname[12], 'x').lower();
1882 funame = curfname[13];
1883 fgid = format(curfname[14], 'x').lower();
1884 fgname = curfname[15];
1885 fid = format(curfname[16], 'x').lower();
1886 finode = format(curfname[17], 'x').lower();
1887 flinkcount = format(curfname[18], 'x').lower();
1888 fdev_minor = format(curfname[19], 'x').lower();
1889 fdev_major = format(curfname[20], 'x').lower();
1890 frdev_minor = format(curfname[21], 'x').lower();
1891 frdev_major = format(curfname[22], 'x').lower();
1892 fseeknextfile = curfname[23];
1893 extradata = curfname[24];
1894 fheaderchecksumtype = curfname[25];
1895 fcontentchecksumtype = curfname[26];
1896 fcontents = curfname[27];
1897 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
1898 fcontents.seek(0, 0);
1899 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1900 if(numfiles>0):
1901 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1902 return fp;
1904 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1905 formatspecs = FormatSpecsListToDict(formatspecs);
1906 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1907 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1909 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1910 formatspecs = FormatSpecsListToDict(formatspecs);
1911 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1912 if(os.path.exists(outfile)):
1913 try:
1914 os.unlink(outfile);
1915 except OSError as e:
1916 pass;
1917 if(outfile=="-"):
1918 verbose = False;
1919 catfpfp = BytesIO();
1920 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1921 catfp = outfile;
1922 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1923 catfp = BytesIO();
1924 else:
1925 fbasename = os.path.splitext(outfile)[0];
1926 fextname = os.path.splitext(outfile)[1];
1927 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1928 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1929 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1930 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1931 try:
1932 catfp.flush();
1933 os.fsync(catfp.fileno());
1934 except io.UnsupportedOperation:
1935 pass;
1936 except AttributeError:
1937 pass;
1938 except OSError as e:
1939 pass;
1940 if(outfile=="-"):
1941 catfp.seek(0, 0);
1942 if(hasattr(sys.stdout, "buffer")):
1943 shutil.copyfileobj(catfp, sys.stdout.buffer);
1944 else:
1945 shutil.copyfileobj(catfp, sys.stdout);
1946 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1947 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1948 catfp.seek(0, 0);
1949 upload_file_to_internet_file(catfp, outfile);
1950 if(returnfp):
1951 catfp.seek(0, 0);
1952 return catfp;
1953 else:
1954 catfp.close();
1955 return True;
1957 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1958 formatspecs = FormatSpecsListToDict(formatspecs);
1959 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1960 if(os.path.exists(outfile)):
1961 try:
1962 os.unlink(outfile);
1963 except OSError as e:
1964 pass;
1965 if(outfile=="-"):
1966 verbose = False;
1967 catfpfp = BytesIO();
1968 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1969 catfp = outfile;
1970 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1971 catfp = BytesIO();
1972 else:
1973 fbasename = os.path.splitext(outfile)[0];
1974 fextname = os.path.splitext(outfile)[1];
1975 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1976 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1977 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1978 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1979 try:
1980 catfp.flush();
1981 os.fsync(catfp.fileno());
1982 except io.UnsupportedOperation:
1983 pass;
1984 except AttributeError:
1985 pass;
1986 except OSError as e:
1987 pass;
1988 if(outfile=="-"):
1989 catfp.seek(0, 0);
1990 if(hasattr(sys.stdout, "buffer")):
1991 shutil.copyfileobj(catfp, sys.stdout.buffer);
1992 else:
1993 shutil.copyfileobj(catfp, sys.stdout);
1994 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1995 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1996 catfp.seek(0, 0);
1997 upload_file_to_internet_file(catfp, outfile);
1998 if(returnfp):
1999 catfp.seek(0, 0);
2000 return catfp;
2001 else:
2002 catfp.close();
2003 return True;
2005 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2006 formatspecs = FormatSpecsListToDict(formatspecs);
2007 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2008 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2010 def PrintPermissionString(fchmode, ftype):
2011 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2012 permissionstr = "";
2013 for fmodval in str(oct(fchmode))[-3:]:
2014 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2015 if(ftype==0 or ftype==7):
2016 permissionstr = "-" + permissionstr;
2017 if(ftype==1):
2018 permissionstr = "h" + permissionstr;
2019 if(ftype==2):
2020 permissionstr = "l" + permissionstr;
2021 if(ftype==3):
2022 permissionstr = "c" + permissionstr;
2023 if(ftype==4):
2024 permissionstr = "b" + permissionstr;
2025 if(ftype==5):
2026 permissionstr = "d" + permissionstr;
2027 if(ftype==6):
2028 permissionstr = "f" + permissionstr;
2029 if(ftype==8):
2030 permissionstr = "D" + permissionstr;
2031 if(ftype==9):
2032 permissionstr = "p" + permissionstr;
2033 if(ftype==10):
2034 permissionstr = "w" + permissionstr;
2035 try:
2036 permissionoutstr = stat.filemode(fchmode);
2037 except AttributeError:
2038 permissionoutstr = permissionstr;
2039 except KeyError:
2040 permissionoutstr = permissionstr;
2041 return permissionoutstr;
2043 def PrintPermissionStringAlt(fchmode, ftype):
2044 permissions = {
2045 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2046 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2048 # Translate file mode into permission string
2049 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2050 # Append file type indicator
2051 type_indicators = {
2052 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2053 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2055 file_type = type_indicators.get(ftype, '-');
2056 permissionstr = file_type + permissionstr;
2057 try:
2058 permissionoutstr = stat.filemode(fchmode);
2059 except AttributeError:
2060 permissionoutstr = permissionstr;
2061 return permissionoutstr;
2063 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2064 formatspecs = FormatSpecsListToDict(formatspecs);
2065 if(hasattr(infile, "read") or hasattr(infile, "write")):
2066 catfp = infile;
2067 else:
2068 try:
2069 catfp = open(infile, "rb");
2070 except FileNotFoundError:
2071 return False;
2072 catfp.seek(0, 0);
2073 prefp = catfp.read(2);
2074 filetype = False;
2075 if(prefp==binascii.unhexlify("1f8b")):
2076 filetype = "gzip";
2077 catfp.seek(0, 0);
2078 prefp = catfp.read(3);
2079 if(prefp==binascii.unhexlify("425a68")):
2080 filetype = "bzip2";
2081 if(prefp==binascii.unhexlify("5d0000")):
2082 filetype = "lzma";
2083 catfp.seek(0, 0);
2084 prefp = catfp.read(4);
2085 if(prefp==binascii.unhexlify("28b52ffd")):
2086 filetype = "zstd";
2087 if(prefp==binascii.unhexlify("04224d18")):
2088 filetype = "lz4";
2089 if(prefp==binascii.unhexlify("504B0304")):
2090 filetype = "zipfile";
2091 catfp.seek(0, 0);
2092 prefp = catfp.read(5);
2093 if(prefp==binascii.unhexlify("7573746172")):
2094 filetype = "tarfile";
2095 catfp.seek(0, 0);
2096 prefp = catfp.read(6);
2097 if(prefp==binascii.unhexlify("fd377a585a00")):
2098 filetype = "lzma";
2099 if(prefp==binascii.unhexlify("377abcaf271c")):
2100 filetype = "7zipfile";
2101 catfp.seek(0, 0);
2102 prefp = catfp.read(7);
2103 if(prefp==binascii.unhexlify("526172211a0700")):
2104 filetype = "rarfile";
2105 if(prefp==binascii.unhexlify("43617446696c65")):
2106 filetype = "catfile";
2107 catfp.seek(0, 0);
2108 prefp = catfp.read(8);
2109 if(prefp==binascii.unhexlify("526172211a070100")):
2110 filetype = "rarfile";
2111 catfp.seek(0, 0);
2112 prefp = catfp.read(formatspecs['format_len']);
2113 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2114 filetype = formatspecs['format_lower'];
2115 catfp.seek(0, 0);
2116 prefp = catfp.read(9);
2117 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2118 filetype = "lzo";
2119 catfp.seek(0, 0);
2120 prefp = catfp.read(10);
2121 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2122 filetype = "tarfile";
2123 catfp.seek(0, 0);
2124 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2125 if(TarFileCheck(catfp)):
2126 filetype = "tarfile";
2127 if(not filetype):
2128 if(TarFileCheck(catfp)):
2129 filetype = "tarfile";
2130 elif(zipfile.is_zipfile(catfp)):
2131 filetype = "zipfile";
2132 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2133 filetype = "rarile";
2134 else:
2135 filetype = False;
2136 catfp.seek(0, 0);
2137 if(closefp):
2138 catfp.close();
2139 return filetype;
2141 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2142 formatspecs = FormatSpecsListToDict(formatspecs);
2143 try:
2144 instringsfile = BytesIO(instring);
2145 except TypeError:
2146 instringsfile = BytesIO(instring.encode("UTF-8"));
2147 return CheckCompressionType(instringsfile, formatspecs, closefp);
2149 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2150 formatspecs = FormatSpecsListToDict(formatspecs);
2151 compresscheck = CheckCompressionType(fp, formatspecs, False);
2152 if(compresscheck=="gzip" or compresscheck=="gz"):
2153 return archivefile_gzip_mimetype;
2154 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2155 return archivefile_bzip2_mimetype;
2156 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2157 return archivefile_zstandard_mimetype;
2158 if(compresscheck=="lz4"):
2159 return archivefile_lz4_mimetype;
2160 if(compresscheck=="lzo" or compresscheck=="lzop"):
2161 return archivefile_lzop_mimetype;
2162 if(compresscheck=="lzma"):
2163 return archivefile_lzma_mimetype;
2164 if(compresscheck=="xz"):
2165 return archivefile_xz_mimetype;
2166 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2167 return archivefile_cat_mimetype;
2168 if(not compresscheck):
2169 return False;
2170 return False;
2172 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2173 formatspecs = FormatSpecsListToDict(formatspecs);
2174 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2175 return False;
2176 compresscheck = CheckCompressionType(fp, formatspecs, False);
2177 if(compresscheck=="gzip"):
2178 try:
2179 import gzip;
2180 except ImportError:
2181 return False;
2182 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2183 if(compresscheck=="bzip2"):
2184 try:
2185 import bz2;
2186 except ImportError:
2187 return False;
2188 catfp = BytesIO();
2189 catfp.write(bz2.decompress(fp.read()));
2190 if(compresscheck=="zstd"):
2191 try:
2192 import zstandard;
2193 except ImportError:
2194 return False;
2195 catfp = BytesIO();
2196 catfp.write(zstandard.decompress(fp.read()));
2197 if(compresscheck=="lz4"):
2198 try:
2199 import lz4.frame;
2200 except ImportError:
2201 return False;
2202 catfp = BytesIO();
2203 catfp.write(lz4.frame.decompress(fp.read()));
2204 if(compresscheck=="lzo" or compresscheck=="lzop"):
2205 try:
2206 import lzo;
2207 except ImportError:
2208 return False;
2209 catfp = BytesIO();
2210 catfp.write(lzo.decompress(fp.read()));
2211 if(compresscheck=="lzma" or compresscheck=="xz"):
2212 try:
2213 import lzma;
2214 except ImportError:
2215 try:
2216 from backports import lzma
2217 except ImportError:
2218 return False;
2219 catfp = BytesIO();
2220 catfp.write(lzma.decompress(fp.read()));
2221 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2222 catfp = fp;
2223 if(not compresscheck):
2224 try:
2225 import lzma;
2226 except ImportError:
2227 try:
2228 from backports import lzma
2229 except ImportError:
2230 return False;
2231 catfp = BytesIO();
2232 with fp as fpcontent:
2233 try:
2234 catfp.write(lzma.decompress(fp.read()));
2235 except lzma.LZMAError:
2236 return False;
2237 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2238 fp.close();
2239 return catfp;
2241 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2243 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2244 formatspecs = FormatSpecsListToDict(formatspecs);
2245 compresscheck = CheckCompressionType(infile, formatspecs, False);
2246 if(sys.version_info[0]==2 and compresscheck):
2247 if(mode=="rt"):
2248 mode = "r";
2249 if(mode=="wt"):
2250 mode = "w";
2251 try:
2252 if(compresscheck=="gzip"):
2253 try:
2254 import gzip;
2255 except ImportError:
2256 return False;
2257 try:
2258 filefp = gzip.open(infile, mode, encoding="UTF-8");
2259 except (ValueError, TypeError) as e:
2260 filefp = gzip.open(infile, mode);
2261 if(compresscheck=="bzip2"):
2262 try:
2263 import bz2;
2264 except ImportError:
2265 return False;
2266 try:
2267 filefp = bz2.open(infile, mode, encoding="UTF-8");
2268 except (ValueError, TypeError) as e:
2269 filefp = bz2.open(infile, mode);
2270 if(compresscheck=="zstd"):
2271 try:
2272 import zstandard;
2273 except ImportError:
2274 return False;
2275 try:
2276 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2277 except (ValueError, TypeError) as e:
2278 filefp = zstandard.open(infile, mode);
2279 if(compresscheck=="lz4"):
2280 try:
2281 import lz4.frame;
2282 except ImportError:
2283 return False;
2284 try:
2285 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2286 except (ValueError, TypeError) as e:
2287 filefp = lz4.frame.open(infile, mode);
2288 if(compresscheck=="lzo"):
2289 try:
2290 import lzo;
2291 except ImportError:
2292 return False;
2293 try:
2294 filefp = lzo.open(infile, mode, encoding="UTF-8");
2295 except (ValueError, TypeError) as e:
2296 filefp = lzo.open(infile, mode);
2297 if(compresscheck=="lzma"):
2298 try:
2299 import lzma;
2300 except ImportError:
2301 try:
2302 from backports import lzma
2303 except ImportError:
2304 return False;
2305 try:
2306 filefp = lzma.open(infile, mode, encoding="UTF-8");
2307 except (ValueError, TypeError) as e:
2308 filefp = lzma.open(infile, mode);
2309 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2310 try:
2311 filefp = open(infile, mode, encoding="UTF-8");
2312 except (ValueError, TypeError) as e:
2313 filefp = open(infile, mode);
2314 if(not compresscheck):
2315 try:
2316 filefp = open(infile, mode, encoding="UTF-8");
2317 except (ValueError, TypeError) as e:
2318 filefp = open(infile, mode);
2319 except FileNotFoundError:
2320 return False;
2321 return filefp;
2323 def UncompressString(infile):
2324 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2325 if(compresscheck=="gzip"):
2326 try:
2327 import gzip;
2328 except ImportError:
2329 return False;
2330 fileuz = gzip.decompress(infile);
2331 if(compresscheck=="bzip2"):
2332 try:
2333 import bz2;
2334 except ImportError:
2335 return False;
2336 fileuz = bz2.decompress(infile);
2337 if(compresscheck=="zstd"):
2338 try:
2339 import zstandard;
2340 except ImportError:
2341 return False;
2342 fileuz = zstandard.decompress(infile);
2343 if(compresscheck=="lz4"):
2344 try:
2345 import lz4.frame;
2346 except ImportError:
2347 return False;
2348 fileuz = lz4.frame.decompress(infile);
2349 if(compresscheck=="lzo"):
2350 try:
2351 import lzo;
2352 except ImportError:
2353 return False;
2354 fileuz = lzo.decompress(infile);
2355 if(compresscheck=="lzma"):
2356 try:
2357 import lzma;
2358 except ImportError:
2359 try:
2360 from backports import lzma
2361 except ImportError:
2362 return False;
2363 fileuz = lzma.decompress(infile);
2364 if(not compresscheck):
2365 fileuz = infile;
2366 if(hasattr(fileuz, 'decode')):
2367 fileuz = fileuz.decode("UTF-8");
2368 return fileuz;
2370 def UncompressStringAlt(infile):
2371 filefp = StringIO();
2372 outstring = UncompressString(infile);
2373 filefp.write(outstring);
2374 filefp.seek(0, 0);
2375 return filefp;
2377 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2378 formatspecs = FormatSpecsListToDict(formatspecs);
2379 compresscheck = CheckCompressionType(infile, formatspecs, False);
2380 if(not compresscheck):
2381 fextname = os.path.splitext(infile)[1];
2382 if(fextname==".gz"):
2383 compresscheck = "gzip";
2384 elif(fextname==".bz2"):
2385 compresscheck = "bzip2";
2386 elif(fextname==".zst"):
2387 compresscheck = "zstd";
2388 elif(fextname==".lz4"):
2389 compresscheck = "lz4";
2390 elif(fextname==".lzo" or fextname==".lzop"):
2391 compresscheck = "lzo";
2392 elif(fextname==".lzma" or fextname==".xz"):
2393 compresscheck = "lzma";
2394 else:
2395 return False;
2396 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2397 if(TarFileCheck(infile)):
2398 filetype = "tarfile";
2399 if(not compresscheck):
2400 if(TarFileCheck(infile)):
2401 return "tarfile";
2402 elif(zipfile.is_zipfile(infile)):
2403 return "zipfile";
2404 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2405 return "rarile";
2406 else:
2407 return False;
2408 return False;
2409 if(compresscheck=="catfile"):
2410 return "catfile";
2411 if(compresscheck==formatspecs['format_lower']):
2412 return formatspecs['format_lower'];
2413 if(compresscheck=="tarfile"):
2414 return "tarfile";
2415 if(compresscheck=="zipfile"):
2416 return "zipfile";
2417 if(rarfile_support and compresscheck=="rarfile"):
2418 return "rarfile";
2419 if(py7zr_support and compresscheck=="7zipfile"):
2420 return "7zipfile";
2421 if(hasattr(infile, "read") or hasattr(infile, "write")):
2422 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2423 else:
2424 try:
2425 if(compresscheck=="gzip"):
2426 try:
2427 import gzip;
2428 except ImportError:
2429 return False;
2430 catfp = gzip.GzipFile(infile, "rb");
2431 if(compresscheck=="bzip2"):
2432 try:
2433 import bz2;
2434 except ImportError:
2435 return False;
2436 catfp = bz2.BZ2File(infile, "rb");
2437 if(compresscheck=="lz4"):
2438 try:
2439 import lz4.frame;
2440 except ImportError:
2441 return False;
2442 catfp = lz4.frame.open(infile, "rb");
2443 if(compresscheck=="zstd"):
2444 try:
2445 import zstandard;
2446 except ImportError:
2447 return False;
2448 catfp = zstandard.open(infile, "rb");
2449 if(compresscheck=="lzma" or compresscheck=="xz"):
2450 try:
2451 import lzma;
2452 except ImportError:
2453 try:
2454 from backports import lzma
2455 except ImportError:
2456 return False;
2457 catfp = lzma.open(infile, "rb");
2458 except FileNotFoundError:
2459 return False;
2460 filetype = False;
2461 prefp = catfp.read(5);
2462 if(prefp==binascii.unhexlify("7573746172")):
2463 filetype = "tarfile";
2464 catfp.seek(0, 0);
2465 prefp = catfp.read(7);
2466 if(prefp==binascii.unhexlify("43617446696c65")):
2467 filetype = "catfile";
2468 catfp.seek(0, 0);
2469 prefp = catfp.read(formatspecs['format_len']);
2470 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2471 filetype = formatspecs['format_lower'];
2472 catfp.seek(0, 0);
2473 prefp = catfp.read(10);
2474 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2475 filetype = "tarfile";
2476 catfp.seek(0, 0);
2477 if(closefp):
2478 catfp.close();
2479 return filetype;
2481 def GZipCompress(data, compresslevel=9):
2482 try:
2483 import gzip;
2484 except ImportError:
2485 return False;
2486 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2487 tmpfp.close();
2488 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2489 tmpfp.write(data);
2490 tmpfp.close();
2491 try:
2492 catfp = open(tmpfp.name, "rb");
2493 except FileNotFoundError:
2494 return False;
2495 catdata = catfp.read();
2496 catfp.close();
2497 return catdata;
2499 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2500 formatspecs = FormatSpecsListToDict(formatspecs);
2501 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2502 return False;
2503 fp.seek(0, 0);
2504 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2505 compression = "auto";
2506 if(compression not in compressionlist and compression is None):
2507 compression = "auto";
2508 if(compression=="gzip"):
2509 try:
2510 import gzip;
2511 except ImportError:
2512 return False;
2513 catfp = BytesIO();
2514 if(compressionlevel is None):
2515 compressionlevel = 9;
2516 else:
2517 compressionlevel = int(compressionlevel);
2518 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2519 if(compression=="bzip2"):
2520 try:
2521 import bz2;
2522 except ImportError:
2523 return False;
2524 catfp = BytesIO();
2525 if(compressionlevel is None):
2526 compressionlevel = 9;
2527 else:
2528 compressionlevel = int(compressionlevel);
2529 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2530 if(compression=="lz4"):
2531 try:
2532 import lz4.frame;
2533 except ImportError:
2534 return False;
2535 catfp = BytesIO();
2536 if(compressionlevel is None):
2537 compressionlevel = 9;
2538 else:
2539 compressionlevel = int(compressionlevel);
2540 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2541 if(compression=="lzo" or compression=="lzop"):
2542 try:
2543 import lzo;
2544 except ImportError:
2545 return False;
2546 catfp = BytesIO();
2547 if(compressionlevel is None):
2548 compressionlevel = 9;
2549 else:
2550 compressionlevel = int(compressionlevel);
2551 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2552 if(compression=="zstd"):
2553 try:
2554 import zstandard;
2555 except ImportError:
2556 return False;
2557 catfp = BytesIO();
2558 if(compressionlevel is None):
2559 compressionlevel = 10;
2560 else:
2561 compressionlevel = int(compressionlevel);
2562 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2563 if(compression=="lzma"):
2564 try:
2565 import lzma;
2566 except ImportError:
2567 try:
2568 from backports import lzma
2569 except ImportError:
2570 return False;
2571 catfp = BytesIO();
2572 if(compressionlevel is None):
2573 compressionlevel = 9;
2574 else:
2575 compressionlevel = int(compressionlevel);
2576 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2577 if(compression=="xz"):
2578 try:
2579 import lzma;
2580 except ImportError:
2581 try:
2582 from backports import lzma
2583 except ImportError:
2584 return False;
2585 catfp = BytesIO();
2586 if(compressionlevel is None):
2587 compressionlevel = 9;
2588 else:
2589 compressionlevel = int(compressionlevel);
2590 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2591 if(compression=="auto" or compression is None):
2592 catfp = fp;
2593 catfp.seek(0, 0);
2594 return catfp;
2596 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2598 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2599 if(outfile is None):
2600 return False;
2601 fbasename = os.path.splitext(outfile)[0];
2602 fextname = os.path.splitext(outfile)[1];
2603 if(compressionlevel is None and fextname!=".zst"):
2604 compressionlevel = 9;
2605 elif(compressionlevel is None and fextname==".zst"):
2606 compressionlevel = 10;
2607 else:
2608 compressionlevel = int(compressionlevel);
2609 if(sys.version_info[0]==2):
2610 mode = "w";
2611 else:
2612 mode = "wb";
2613 try:
2614 if(fextname not in outextlistwd or not compressionenable):
2615 try:
2616 outfp = open(outfile, "wb", encoding="UTF-8");
2617 except (ValueError, TypeError) as e:
2618 outfp = open(outfile, "wb");
2619 elif(fextname==".gz"):
2620 try:
2621 import gzip;
2622 except ImportError:
2623 return False;
2624 try:
2625 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2626 except (ValueError, TypeError) as e:
2627 outfp = gzip.open(outfile, mode, compressionlevel);
2628 elif(fextname==".bz2"):
2629 try:
2630 import bz2;
2631 except ImportError:
2632 return False;
2633 try:
2634 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2635 except (ValueError, TypeError) as e:
2636 outfp = bz2.open(outfile, mode, compressionlevel);
2637 elif(fextname==".zst"):
2638 try:
2639 import zstandard;
2640 except ImportError:
2641 return False;
2642 try:
2643 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2644 except (ValueError, TypeError) as e:
2645 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2646 elif(fextname==".xz"):
2647 try:
2648 import lzma;
2649 except ImportError:
2650 try:
2651 from backports import lzma
2652 except ImportError:
2653 return False;
2654 try:
2655 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2656 except (ValueError, TypeError) as e:
2657 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2658 elif(fextname==".lz4"):
2659 try:
2660 import lz4.frame;
2661 except ImportError:
2662 return False;
2663 try:
2664 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2665 except (ValueError, TypeError) as e:
2666 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2667 elif(fextname==".lzo"):
2668 try:
2669 import lzo;
2670 except ImportError:
2671 return False;
2672 try:
2673 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2674 except (ValueError, TypeError) as e:
2675 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2676 elif(fextname==".lzma"):
2677 try:
2678 import lzma;
2679 except ImportError:
2680 try:
2681 from backports import lzma
2682 except ImportError:
2683 return False;
2684 try:
2685 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2686 except (ValueError, TypeError) as e:
2687 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2688 except FileNotFoundError:
2689 return False;
2690 return outfp;
2692 def GetDevMajorMinor(fdev):
2693 retdev = [];
2694 if(hasattr(os, "minor")):
2695 retdev.append(os.minor(fdev));
2696 else:
2697 retdev.append(0);
2698 if(hasattr(os, "major")):
2699 retdev.append(os.major(fdev));
2700 else:
2701 retdev.append(0);
2702 return retdev;
2704 def CheckSumSupport(checkfor, guaranteed=True):
2705 if(guaranteed):
2706 try:
2707 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2708 except AttributeError:
2709 hash_list = sorted(list(hashlib.algorithms));
2710 else:
2711 try:
2712 hash_list = sorted(list(hashlib.algorithms_available));
2713 except AttributeError:
2714 hash_list = sorted(list(hashlib.algorithms));
2715 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2716 if(checkfor in checklistout):
2717 return True;
2718 else:
2719 return False;
2721 def CheckSumSupportAlt(checkfor, guaranteed=True):
2722 if(guaranteed):
2723 try:
2724 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2725 except AttributeError:
2726 hash_list = sorted(list(hashlib.algorithms));
2727 else:
2728 try:
2729 hash_list = sorted(list(hashlib.algorithms_available));
2730 except AttributeError:
2731 hash_list = sorted(list(hashlib.algorithms));
2732 checklistout = hash_list;
2733 if(checkfor in checklistout):
2734 return True;
2735 else:
2736 return False;
2738 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2739 formatspecs = FormatSpecsListToDict(formatspecs);
2740 advancedlist = formatspecs['use_advanced_list'];
2741 altinode = formatspecs['use_alt_inode'];
2742 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2743 outfile = RemoveWindowsPath(outfile);
2744 checksumtype = checksumtype.lower();
2745 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2746 checksumtype="crc32";
2747 if(checksumtype=="none"):
2748 checksumtype = "";
2749 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2750 compression = "auto";
2751 if(compression not in compressionlist and compression is None):
2752 compression = "auto";
2753 if(verbose):
2754 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2755 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2756 if(os.path.exists(outfile)):
2757 try:
2758 os.unlink(outfile);
2759 except OSError as e:
2760 pass;
2761 if(outfile=="-"):
2762 verbose = False;
2763 catfp = BytesIO();
2764 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2765 catfp = outfile;
2766 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2767 catfp = BytesIO();
2768 else:
2769 fbasename = os.path.splitext(outfile)[0];
2770 fextname = os.path.splitext(outfile)[1];
2771 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2772 catver = formatspecs['format_ver'];
2773 fileheaderver = str(int(catver.replace(".", "")));
2774 infilelist = [];
2775 if(infiles=="-"):
2776 for line in sys.stdin:
2777 infilelist.append(line.strip());
2778 infilelist = list(filter(None, infilelist));
2779 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2780 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2781 return False;
2782 with UncompressFile(infiles, formatspecs, "r") as finfile:
2783 for line in finfile:
2784 infilelist.append(line.strip());
2785 infilelist = list(filter(None, infilelist));
2786 else:
2787 if(isinstance(infiles, (list, tuple, ))):
2788 infilelist = list(filter(None, infiles));
2789 elif(isinstance(infiles, (str, ))):
2790 infilelist = list(filter(None, [infiles]));
2791 if(advancedlist):
2792 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2793 else:
2794 GetDirList = ListDir(infilelist, followlink, False);
2795 if(not GetDirList):
2796 return False;
2797 curinode = 0;
2798 curfid = 0;
2799 inodelist = [];
2800 inodetofile = {};
2801 filetoinode = {};
2802 inodetocatinode = {};
2803 numfiles = int(len(GetDirList));
2804 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2805 for curfname in GetDirList:
2806 if(re.findall(r"^[.|/]", curfname)):
2807 fname = curfname;
2808 else:
2809 fname = "./"+curfname;
2810 if(verbose):
2811 VerbosePrintOut(fname);
2812 if(not followlink or followlink is None):
2813 fstatinfo = os.lstat(fname);
2814 else:
2815 fstatinfo = os.stat(fname);
2816 fpremode = fstatinfo.st_mode;
2817 finode = fstatinfo.st_ino;
2818 flinkcount = fstatinfo.st_nlink;
2819 ftype = 0;
2820 if(stat.S_ISREG(fpremode)):
2821 ftype = 0;
2822 elif(stat.S_ISLNK(fpremode)):
2823 ftype = 2;
2824 elif(stat.S_ISCHR(fpremode)):
2825 ftype = 3;
2826 elif(stat.S_ISBLK(fpremode)):
2827 ftype = 4;
2828 elif(stat.S_ISDIR(fpremode)):
2829 ftype = 5;
2830 elif(stat.S_ISFIFO(fpremode)):
2831 ftype = 6;
2832 elif(stat.S_ISSOCK(fpremode)):
2833 ftype = 8;
2834 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2835 ftype = 9;
2836 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2837 ftype = 10;
2838 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2839 ftype = 11;
2840 else:
2841 ftype = 0;
2842 flinkname = "";
2843 fcurfid = format(int(curfid), 'x').lower();
2844 if(not followlink and finode!=0):
2845 if(ftype!=1):
2846 if(finode in inodelist):
2847 ftype = 1;
2848 flinkname = inodetofile[finode];
2849 if(altinode):
2850 fcurinode = format(int(finode), 'x').lower();
2851 else:
2852 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2853 if(finode not in inodelist):
2854 inodelist.append(finode);
2855 inodetofile.update({finode: fname});
2856 inodetocatinode.update({finode: curinode});
2857 if(altinode):
2858 fcurinode = format(int(finode), 'x').lower();
2859 else:
2860 fcurinode = format(int(curinode), 'x').lower();
2861 curinode = curinode + 1;
2862 else:
2863 fcurinode = format(int(curinode), 'x').lower();
2864 curinode = curinode + 1;
2865 curfid = curfid + 1;
2866 if(ftype==2):
2867 flinkname = os.readlink(fname);
2868 fdev = fstatinfo.st_dev;
2869 getfdev = GetDevMajorMinor(fdev);
2870 fdev_minor = getfdev[0];
2871 fdev_major = getfdev[1];
2872 frdev = fstatinfo.st_dev;
2873 if(hasattr(fstatinfo, "st_rdev")):
2874 frdev = fstatinfo.st_rdev;
2875 else:
2876 frdev = fstatinfo.st_dev;
2877 getfrdev = GetDevMajorMinor(frdev);
2878 frdev_minor = getfrdev[0];
2879 frdev_major = getfrdev[1];
2880 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2881 fsize = format(int("0"), 'x').lower();
2882 elif(ftype==0 or ftype==7):
2883 fsize = format(int(fstatinfo.st_size), 'x').lower();
2884 else:
2885 fsize = format(int(fstatinfo.st_size)).lower();
2886 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2887 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2888 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2889 if(hasattr(fstatinfo, "st_birthtime")):
2890 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2891 else:
2892 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2893 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2894 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2895 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2896 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2897 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2898 funame = "";
2899 try:
2900 import pwd;
2901 try:
2902 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2903 funame = userinfo.pw_name;
2904 except KeyError:
2905 funame = "";
2906 except ImportError:
2907 funame = "";
2908 fgname = "";
2909 try:
2910 import grp;
2911 try:
2912 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2913 fgname = groupinfo.gr_name;
2914 except KeyError:
2915 fgname = "";
2916 except ImportError:
2917 fgname = "";
2918 fdev_minor = format(int(fdev_minor), 'x').lower();
2919 fdev_major = format(int(fdev_major), 'x').lower();
2920 frdev_minor = format(int(frdev_minor), 'x').lower();
2921 frdev_major = format(int(frdev_major), 'x').lower();
2922 finode = format(int(finode), 'x').lower();
2923 flinkcount = format(int(flinkcount), 'x').lower();
2924 if(hasattr(fstatinfo, "st_file_attributes")):
2925 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2926 else:
2927 fwinattributes = format(int(0), 'x').lower();
2928 fcompression = "";
2929 fcsize = format(int(0), 'x').lower();
2930 fcontents = BytesIO();
2931 if(ftype==0 or ftype==7):
2932 with open(fname, "rb") as fpc:
2933 shutil.copyfileobj(fpc, fcontents);
2934 if(not compresswholefile):
2935 fcontents.seek(0, 2);
2936 ucfsize = fcontents.tell();
2937 fcontents.seek(0, 0);
2938 if(compression=="auto"):
2939 ilsize = len(compressionlistalt);
2940 ilmin = 0;
2941 ilcsize = [];
2942 while(ilmin < ilsize):
2943 cfcontents = BytesIO();
2944 shutil.copyfileobj(fcontents, cfcontents);
2945 fcontents.seek(0, 0);
2946 cfcontents.seek(0, 0);
2947 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2948 if(cfcontents):
2949 cfcontents.seek(0, 2);
2950 ilcsize.append(cfcontents.tell());
2951 cfcontents.close();
2952 else:
2953 try:
2954 ilcsize.append(sys.maxint);
2955 except AttributeError:
2956 ilcsize.append(sys.maxsize);
2957 ilmin = ilmin + 1;
2958 ilcmin = ilcsize.index(min(ilcsize));
2959 compression = compressionlistalt[ilcmin];
2960 fcontents.seek(0, 0);
2961 cfcontents = BytesIO();
2962 shutil.copyfileobj(fcontents, cfcontents);
2963 cfcontents.seek(0, 0);
2964 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2965 cfcontents.seek(0, 2);
2966 cfsize = cfcontents.tell();
2967 if(ucfsize > cfsize):
2968 fcsize = format(int(cfsize), 'x').lower();
2969 fcompression = compression;
2970 fcontents.close();
2971 fcontents = cfcontents;
2972 if(fcompression=="none"):
2973 fcompression = "";
2974 if(followlink and (ftype==1 or ftype==2)):
2975 flstatinfo = os.stat(flinkname);
2976 with open(flinkname, "rb") as fpc:
2977 shutil.copyfileobj(fpc, fcontents);
2978 if(not compresswholefile):
2979 fcontents.seek(0, 2);
2980 ucfsize = fcontents.tell();
2981 fcontents.seek(0, 0);
2982 if(compression=="auto"):
2983 ilsize = len(compressionlistalt);
2984 ilmin = 0;
2985 ilcsize = [];
2986 while(ilmin < ilsize):
2987 cfcontents = BytesIO();
2988 shutil.copyfileobj(fcontents, cfcontents);
2989 fcontents.seek(0, 0);
2990 cfcontents.seek(0, 0);
2991 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2992 if(cfcontents):
2993 cfcontents.seek(0, 2);
2994 ilcsize.append(cfcontents.tell());
2995 cfcontents.close();
2996 else:
2997 try:
2998 ilcsize.append(sys.maxint);
2999 except AttributeError:
3000 ilcsize.append(sys.maxsize);
3001 ilmin = ilmin + 1;
3002 ilcmin = ilcsize.index(min(ilcsize));
3003 compression = compressionlistalt[ilcmin];
3004 fcontents.seek(0, 0);
3005 cfcontents = BytesIO();
3006 shutil.copyfileobj(fcontents, cfcontents);
3007 cfcontents.seek(0, 0);
3008 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3009 cfcontents.seek(0, 2);
3010 cfsize = cfcontents.tell();
3011 if(ucfsize > cfsize):
3012 fcsize = format(int(cfsize), 'x').lower();
3013 fcompression = compression;
3014 fcontents.close();
3015 fcontents = cfcontents;
3016 fcontents.seek(0, 0);
3017 ftypehex = format(ftype, 'x').lower();
3018 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3019 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3020 fcontents.close();
3021 if(numfiles>0):
3022 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3023 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3024 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3025 try:
3026 catfp.flush();
3027 os.fsync(catfp.fileno());
3028 except io.UnsupportedOperation:
3029 pass;
3030 except AttributeError:
3031 pass;
3032 except OSError as e:
3033 pass;
3034 if(outfile=="-"):
3035 catfp.seek(0, 0);
3036 if(hasattr(sys.stdout, "buffer")):
3037 shutil.copyfileobj(catfp, sys.stdout.buffer);
3038 else:
3039 shutil.copyfileobj(catfp, sys.stdout);
3040 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3041 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3042 catfp.seek(0, 0);
3043 upload_file_to_internet_file(catfp, outfile);
3044 if(returnfp):
3045 catfp.seek(0, 0);
3046 return catfp;
3047 else:
3048 catfp.close();
3049 return True;
3051 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3053 if(hasattr(shutil, "register_archive_format")):
3054 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3055 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3056 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3058 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3059 formatspecs = FormatSpecsListToDict(formatspecs);
3060 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3062 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3064 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3065 formatspecs = FormatSpecsListToDict(formatspecs);
3066 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3067 outfile = RemoveWindowsPath(outfile);
3068 checksumtype = checksumtype.lower();
3069 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3070 checksumtype="crc32";
3071 if(checksumtype=="none"):
3072 checksumtype = "";
3073 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3074 compression = "auto";
3075 if(compression not in compressionlist and compression is None):
3076 compression = "auto";
3077 if(verbose):
3078 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3079 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3080 if(os.path.exists(outfile)):
3081 try:
3082 os.unlink(outfile);
3083 except OSError as e:
3084 pass;
3085 if(outfile=="-"):
3086 verbose = False;
3087 catfp = BytesIO();
3088 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3089 catfp = outfile;
3090 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3091 catfp = BytesIO();
3092 else:
3093 fbasename = os.path.splitext(outfile)[0];
3094 fextname = os.path.splitext(outfile)[1];
3095 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3096 catver = formatspecs['format_ver'];
3097 fileheaderver = str(int(catver.replace(".", "")));
3098 curinode = 0;
3099 curfid = 0;
3100 inodelist = [];
3101 inodetofile = {};
3102 filetoinode = {};
3103 inodetocatinode = {};
3104 if(infile=="-"):
3105 infile = BytesIO();
3106 if(hasattr(sys.stdin, "buffer")):
3107 shutil.copyfileobj(sys.stdin.buffer, infile);
3108 else:
3109 shutil.copyfileobj(sys.stdin, infile);
3110 infile.seek(0, 0);
3111 if(not infile):
3112 return False;
3113 infile.seek(0, 0);
3114 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3115 infile = download_file_from_internet_file(infile);
3116 infile.seek(0, 0);
3117 if(not infile):
3118 return False;
3119 infile.seek(0, 0);
3120 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3121 return False;
3122 elif(os.path.exists(infile) and os.path.isfile(infile)):
3123 try:
3124 if(not tarfile.TarFileCheck(infile)):
3125 return False;
3126 except AttributeError:
3127 if(not TarFileCheck(infile)):
3128 return False;
3129 else:
3130 return False;
3131 try:
3132 if(hasattr(infile, "read") or hasattr(infile, "write")):
3133 tarfp = tarfile.open(fileobj=infile, mode="r");
3134 else:
3135 tarfp = tarfile.open(infile, "r");
3136 except FileNotFoundError:
3137 return False;
3138 numfiles = int(len(tarfp.getmembers()));
3139 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3140 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3141 if(re.findall(r"^[.|/]", member.name)):
3142 fname = member.name;
3143 else:
3144 fname = "./"+member.name;
3145 if(verbose):
3146 VerbosePrintOut(fname);
3147 fpremode = member.mode;
3148 ffullmode = member.mode;
3149 flinkcount = 0;
3150 ftype = 0;
3151 if(member.isreg()):
3152 ffullmode = member.mode + stat.S_IFREG;
3153 ftype = 0;
3154 elif(member.isdev()):
3155 ffullmode = member.mode;
3156 ftype = 7;
3157 elif(member.islnk()):
3158 ffullmode = member.mode + stat.S_IFREG;
3159 ftype = 1;
3160 elif(member.issym()):
3161 ffullmode = member.mode + stat.S_IFLNK;
3162 ftype = 2;
3163 elif(member.ischr()):
3164 ffullmode = member.mode + stat.S_IFCHR;
3165 ftype = 3;
3166 elif(member.isblk()):
3167 ffullmode = member.mode + stat.S_IFBLK;
3168 ftype = 4;
3169 elif(member.isdir()):
3170 ffullmode = member.mode + stat.S_IFDIR;
3171 ftype = 5;
3172 elif(member.isfifo()):
3173 ffullmode = member.mode + stat.S_IFIFO;
3174 ftype = 6;
3175 elif(member.issparse()):
3176 ffullmode = member.mode;
3177 ftype = 12;
3178 else:
3179 ffullmode = member.mode;
3180 ftype = 0;
3181 flinkname = "";
3182 fcurfid = format(int(curfid), 'x').lower();
3183 fcurinode = format(int(curfid), 'x').lower();
3184 curfid = curfid + 1;
3185 if(ftype==2):
3186 flinkname = member.linkname;
3187 fdev_minor = format(int(member.devminor), 'x').lower();
3188 fdev_major = format(int(member.devmajor), 'x').lower();
3189 frdev_minor = format(int(member.devminor), 'x').lower();
3190 frdev_major = format(int(member.devmajor), 'x').lower();
3191 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3192 fsize = format(int("0"), 'x').lower();
3193 elif(ftype==0 or ftype==7):
3194 fsize = format(int(member.size), 'x').lower();
3195 else:
3196 fsize = format(int(member.size), 'x').lower();
3197 fatime = format(int(member.mtime), 'x').lower();
3198 fmtime = format(int(member.mtime), 'x').lower();
3199 fctime = format(int(member.mtime), 'x').lower();
3200 fbtime = format(int(member.mtime), 'x').lower();
3201 fmode = format(int(ffullmode), 'x').lower();
3202 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3203 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3204 fuid = format(int(member.uid), 'x').lower();
3205 fgid = format(int(member.gid), 'x').lower();
3206 funame = member.uname;
3207 fgname = member.gname;
3208 flinkcount = format(int(flinkcount), 'x').lower();
3209 fwinattributes = format(int(0), 'x').lower();
3210 fcompression = "";
3211 fcsize = format(int(0), 'x').lower();
3212 fcontents = BytesIO();
3213 if(ftype==0 or ftype==7):
3214 with tarfp.extractfile(member) as fpc:
3215 shutil.copyfileobj(fpc, fcontents);
3216 if(not compresswholefile):
3217 fcontents.seek(0, 2);
3218 ucfsize = fcontents.tell();
3219 fcontents.seek(0, 0);
3220 if(compression=="auto"):
3221 ilsize = len(compressionlistalt);
3222 ilmin = 0;
3223 ilcsize = [];
3224 while(ilmin < ilsize):
3225 cfcontents = BytesIO();
3226 shutil.copyfileobj(fcontents, cfcontents);
3227 fcontents.seek(0, 0);
3228 cfcontents.seek(0, 0);
3229 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3230 if(cfcontents):
3231 cfcontents.seek(0, 2);
3232 ilcsize.append(cfcontents.tell());
3233 cfcontents.close();
3234 else:
3235 try:
3236 ilcsize.append(sys.maxint);
3237 except AttributeError:
3238 ilcsize.append(sys.maxsize);
3239 ilmin = ilmin + 1;
3240 ilcmin = ilcsize.index(min(ilcsize));
3241 compression = compressionlistalt[ilcmin];
3242 fcontents.seek(0, 0);
3243 cfcontents = BytesIO();
3244 shutil.copyfileobj(fcontents, cfcontents);
3245 cfcontents.seek(0, 0);
3246 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3247 cfcontents.seek(0, 2);
3248 cfsize = cfcontents.tell();
3249 if(ucfsize > cfsize):
3250 fcsize = format(int(cfsize), 'x').lower();
3251 fcompression = compression;
3252 fcontents.close();
3253 fcontents = cfcontents;
3254 if(fcompression=="none"):
3255 fcompression = "";
3256 fcontents.seek(0, 0);
3257 ftypehex = format(ftype, 'x').lower();
3258 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3259 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3260 fcontents.close();
3261 if(numfiles>0):
3262 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3263 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3264 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3265 try:
3266 catfp.flush();
3267 os.fsync(catfp.fileno());
3268 except io.UnsupportedOperation:
3269 pass;
3270 except AttributeError:
3271 pass;
3272 except OSError as e:
3273 pass;
3274 if(outfile=="-"):
3275 catfp.seek(0, 0);
3276 if(hasattr(sys.stdout, "buffer")):
3277 shutil.copyfileobj(catfp, sys.stdout.buffer);
3278 else:
3279 shutil.copyfileobj(catfp, sys.stdout);
3280 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3281 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3282 catfp.seek(0, 0);
3283 upload_file_to_internet_file(catfp, outfile);
3284 if(returnfp):
3285 catfp.seek(0, 0);
3286 return catfp;
3287 else:
3288 catfp.close();
3289 return True;
3291 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3293 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3294 formatspecs = FormatSpecsListToDict(formatspecs);
3295 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3296 outfile = RemoveWindowsPath(outfile);
3297 checksumtype = checksumtype.lower();
3298 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3299 checksumtype="crc32";
3300 if(checksumtype=="none"):
3301 checksumtype = "";
3302 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3303 compression = "auto";
3304 if(compression not in compressionlist and compression is None):
3305 compression = "auto";
3306 if(verbose):
3307 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3308 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3309 if(os.path.exists(outfile)):
3310 try:
3311 os.unlink(outfile);
3312 except OSError as e:
3313 pass;
3314 if(outfile=="-"):
3315 verbose = False;
3316 catfp = BytesIO();
3317 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3318 catfp = outfile;
3319 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3320 catfp = BytesIO();
3321 else:
3322 fbasename = os.path.splitext(outfile)[0];
3323 fextname = os.path.splitext(outfile)[1];
3324 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3325 catver = formatspecs['format_ver'];
3326 fileheaderver = str(int(catver.replace(".", "")));
3327 curinode = 0;
3328 curfid = 0;
3329 inodelist = [];
3330 inodetofile = {};
3331 filetoinode = {};
3332 inodetocatinode = {};
3333 if(infile=="-"):
3334 infile = BytesIO();
3335 if(hasattr(sys.stdin, "buffer")):
3336 shutil.copyfileobj(sys.stdin.buffer, infile);
3337 else:
3338 shutil.copyfileobj(sys.stdin, infile);
3339 infile.seek(0, 0);
3340 if(not infile):
3341 return False;
3342 infile.seek(0, 0);
3343 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3344 infile = download_file_from_internet_file(infile);
3345 infile.seek(0, 0);
3346 if(not infile):
3347 return False;
3348 infile.seek(0, 0);
3349 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3350 return False;
3351 else:
3352 return False;
3353 if(not zipfile.is_zipfile(infile)):
3354 return False;
3355 try:
3356 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3357 except FileNotFoundError:
3358 return False;
3359 ziptest = zipfp.testzip();
3360 if(ziptest):
3361 VerbosePrintOut("Bad file found!");
3362 numfiles = int(len(zipfp.infolist()));
3363 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3364 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3365 if(re.findall(r"^[.|/]", member.filename)):
3366 fname = member.filename;
3367 else:
3368 fname = "./"+member.filename;
3369 zipinfo = zipfp.getinfo(member.filename);
3370 if(verbose):
3371 VerbosePrintOut(fname);
3372 if(not member.is_dir()):
3373 fpremode = int(stat.S_IFREG + 438);
3374 elif(member.is_dir()):
3375 fpremode = int(stat.S_IFDIR + 511);
3376 flinkcount = 0;
3377 ftype = 0;
3378 if(not member.is_dir()):
3379 ftype = 0;
3380 elif(member.is_dir()):
3381 ftype = 5;
3382 flinkname = "";
3383 fcurfid = format(int(curfid), 'x').lower();
3384 fcurinode = format(int(curfid), 'x').lower();
3385 curfid = curfid + 1;
3386 fdev_minor = format(int(0), 'x').lower();
3387 fdev_major = format(int(0), 'x').lower();
3388 frdev_minor = format(int(0), 'x').lower();
3389 frdev_major = format(int(0), 'x').lower();
3390 if(ftype==5):
3391 fsize = format(int("0"), 'x').lower();
3392 elif(ftype==0):
3393 fsize = format(int(member.file_size), 'x').lower();
3394 else:
3395 fsize = format(int(member.file_size), 'x').lower();
3396 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3397 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3398 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3399 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3400 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3401 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3402 if(not member.is_dir()):
3403 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3404 fchmode = stat.S_IMODE(fmode);
3405 ftypemod = stat.S_IFMT(fmode);
3406 elif(member.is_dir()):
3407 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3408 fchmode = stat.S_IMODE(fmode);
3409 ftypemod = stat.S_IFMT(fmode);
3410 elif(zipinfo.create_system==3):
3411 fwinattributes = format(int(0), 'x').lower();
3412 fmode = format(int(zipinfo.external_attr), 'x').lower();
3413 fchmode = stat.S_IMODE(fmode);
3414 ftypemod = stat.S_IFMT(fmode);
3415 else:
3416 fwinattributes = format(int(0), 'x').lower();
3417 if(not member.is_dir()):
3418 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3419 fchmode = stat.S_IMODE(fmode);
3420 ftypemod = stat.S_IFMT(fmode);
3421 elif(member.is_dir()):
3422 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3423 fchmode = stat.S_IMODE(fmode);
3424 ftypemod = stat.S_IFMT(fmode);
3425 fcompression = "";
3426 fcsize = format(int(0), 'x').lower();
3427 try:
3428 fuid = format(int(os.getuid()), 'x').lower();
3429 except AttributeError:
3430 fuid = format(int(0), 'x').lower();
3431 except KeyError:
3432 fuid = format(int(0), 'x').lower();
3433 try:
3434 fgid = format(int(os.getgid()), 'x').lower();
3435 except AttributeError:
3436 fgid = format(int(0), 'x').lower();
3437 except KeyError:
3438 fgid = format(int(0), 'x').lower();
3439 try:
3440 import pwd;
3441 try:
3442 userinfo = pwd.getpwuid(os.getuid());
3443 funame = userinfo.pw_name;
3444 except KeyError:
3445 funame = "";
3446 except AttributeError:
3447 funame = "";
3448 except ImportError:
3449 funame = "";
3450 fgname = "";
3451 try:
3452 import grp;
3453 try:
3454 groupinfo = grp.getgrgid(os.getgid());
3455 fgname = groupinfo.gr_name;
3456 except KeyError:
3457 fgname = "";
3458 except AttributeError:
3459 fgname = "";
3460 except ImportError:
3461 fgname = "";
3462 fcontents = BytesIO();
3463 if(ftype==0):
3464 fcontents.write(zipfp.read(member.filename));
3465 if(not compresswholefile):
3466 fcontents.seek(0, 2);
3467 ucfsize = fcontents.tell();
3468 fcontents.seek(0, 0);
3469 if(compression=="auto"):
3470 ilsize = len(compressionlistalt);
3471 ilmin = 0;
3472 ilcsize = [];
3473 while(ilmin < ilsize):
3474 cfcontents = BytesIO();
3475 shutil.copyfileobj(fcontents, cfcontents);
3476 fcontents.seek(0, 0);
3477 cfcontents.seek(0, 0);
3478 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3479 cfcontents.seek(0, 2);
3480 ilcsize.append(cfcontents.tell());
3481 cfcontents.close();
3482 ilmin = ilmin + 1;
3483 ilcmin = ilcsize.index(min(ilcsize));
3484 compression = compressionlistalt[ilcmin];
3485 fcontents.seek(0, 0);
3486 cfcontents = BytesIO();
3487 shutil.copyfileobj(fcontents, cfcontents);
3488 cfcontents.seek(0, 0);
3489 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3490 cfcontents.seek(0, 2);
3491 cfsize = cfcontents.tell();
3492 if(ucfsize > cfsize):
3493 fcsize = format(int(cfsize), 'x').lower();
3494 fcompression = compression;
3495 fcontents.close();
3496 fcontents = cfcontents;
3497 if(fcompression=="none"):
3498 fcompression = "";
3499 fcontents.seek(0, 0);
3500 ftypehex = format(ftype, 'x').lower();
3501 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3502 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3503 fcontents.close();
3504 if(numfiles>0):
3505 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3506 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3507 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3508 try:
3509 catfp.flush();
3510 os.fsync(catfp.fileno());
3511 except io.UnsupportedOperation:
3512 pass;
3513 except AttributeError:
3514 pass;
3515 except OSError as e:
3516 pass;
3517 if(outfile=="-"):
3518 catfp.seek(0, 0);
3519 if(hasattr(sys.stdout, "buffer")):
3520 shutil.copyfileobj(catfp, sys.stdout.buffer);
3521 else:
3522 shutil.copyfileobj(catfp, sys.stdout);
3523 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3524 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3525 catfp.seek(0, 0);
3526 upload_file_to_internet_file(catfp, outfile);
3527 if(returnfp):
3528 catfp.seek(0, 0);
3529 return catfp;
3530 else:
3531 catfp.close();
3532 return True;
3534 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3536 if(not rarfile_support):
3537 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3538 return False
3540 if(rarfile_support):
3541 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3542 formatspecs = FormatSpecsListToDict(formatspecs);
3543 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3544 outfile = RemoveWindowsPath(outfile);
3545 checksumtype = checksumtype.lower();
3546 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3547 checksumtype="crc32";
3548 if(checksumtype=="none"):
3549 checksumtype = "";
3550 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3551 compression = "auto";
3552 if(compression not in compressionlist and compression is None):
3553 compression = "auto";
3554 if(verbose):
3555 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3556 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3557 if(os.path.exists(outfile)):
3558 try:
3559 os.unlink(outfile);
3560 except OSError as e:
3561 pass;
3562 if(outfile=="-"):
3563 verbose = False;
3564 catfp = BytesIO();
3565 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3566 catfp = outfile;
3567 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3568 catfp = BytesIO();
3569 else:
3570 fbasename = os.path.splitext(outfile)[0];
3571 fextname = os.path.splitext(outfile)[1];
3572 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3573 catver = formatspecs['format_ver'];
3574 fileheaderver = str(int(catver.replace(".", "")));
3575 curinode = 0;
3576 curfid = 0;
3577 inodelist = [];
3578 inodetofile = {};
3579 filetoinode = {};
3580 inodetocatinode = {};
3581 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3582 return False;
3583 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3584 return False;
3585 rarfp = rarfile.RarFile(infile, "r");
3586 rartest = rarfp.testrar();
3587 if(rartest):
3588 VerbosePrintOut("Bad file found!");
3589 numfiles = int(len(rarfp.infolist()));
3590 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3591 try:
3592 catfp.flush();
3593 os.fsync(catfp.fileno());
3594 except io.UnsupportedOperation:
3595 pass;
3596 except AttributeError:
3597 pass;
3598 except OSError as e:
3599 pass;
3600 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3601 is_unix = False;
3602 is_windows = False;
3603 if(member.host_os==rarfile.RAR_OS_UNIX):
3604 is_windows = False;
3605 try:
3606 member.external_attr
3607 is_unix = True;
3608 except AttributeError:
3609 is_unix = False;
3610 elif(member.host_os==rarfile.RAR_OS_WIN32):
3611 is_unix = False;
3612 try:
3613 member.external_attr
3614 is_windows = True;
3615 except AttributeError:
3616 is_windows = False;
3617 else:
3618 is_unix = False;
3619 is_windows = False;
3620 if(re.findall(r"^[.|/]", member.filename)):
3621 fname = member.filename;
3622 else:
3623 fname = "./"+member.filename;
3624 rarinfo = rarfp.getinfo(member.filename);
3625 if(verbose):
3626 VerbosePrintOut(fname);
3627 if(is_unix and member.external_attr !=0):
3628 fpremode = int(member.external_attr);
3629 elif(member.is_file()):
3630 fpremode = int(stat.S_IFREG + 438);
3631 elif(member.is_symlink()):
3632 fpremode = int(stat.S_IFLNK + 438);
3633 elif(member.is_dir()):
3634 fpremode = int(stat.S_IFDIR + 511);
3635 if(is_windows and member.external_attr !=0):
3636 fwinattributes = format(int(member.external_attr), 'x').lower();
3637 else:
3638 fwinattributes = format(int(0), 'x').lower();
3639 fcompression = "";
3640 fcsize = format(int(0), 'x').lower();
3641 flinkcount = 0;
3642 ftype = 0;
3643 if(member.is_file()):
3644 ftype = 0;
3645 elif(member.is_symlink()):
3646 ftype = 2;
3647 elif(member.is_dir()):
3648 ftype = 5;
3649 flinkname = "";
3650 if(ftype==2):
3651 flinkname = rarfp.read(member.filename).decode("UTF-8");
3652 fcurfid = format(int(curfid), 'x').lower();
3653 fcurinode = format(int(curfid), 'x').lower();
3654 curfid = curfid + 1;
3655 fdev_minor = format(int(0), 'x').lower();
3656 fdev_major = format(int(0), 'x').lower();
3657 frdev_minor = format(int(0), 'x').lower();
3658 frdev_major = format(int(0), 'x').lower();
3659 if(ftype==5):
3660 fsize = format(int("0"), 'x').lower();
3661 elif(ftype==0):
3662 fsize = format(int(member.file_size), 'x').lower();
3663 else:
3664 fsize = format(int(member.file_size), 'x').lower();
3665 try:
3666 if(member.atime):
3667 fatime = format(int(member.atime.timestamp()), 'x').lower();
3668 else:
3669 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3670 except AttributeError:
3671 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3672 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3673 try:
3674 if(member.ctime):
3675 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3676 else:
3677 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3678 except AttributeError:
3679 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3680 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3681 if(is_unix and member.external_attr !=0):
3682 fmode = format(int(member.external_attr), 'x').lower();
3683 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3684 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3685 elif(member.is_file()):
3686 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3687 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3688 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3689 elif(member.is_symlink()):
3690 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3691 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3692 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3693 elif(member.is_dir()):
3694 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3695 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3696 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3697 try:
3698 fuid = format(int(os.getuid()), 'x').lower();
3699 except AttributeError:
3700 fuid = format(int(0), 'x').lower();
3701 except KeyError:
3702 fuid = format(int(0), 'x').lower();
3703 try:
3704 fgid = format(int(os.getgid()), 'x').lower();
3705 except AttributeError:
3706 fgid = format(int(0), 'x').lower();
3707 except KeyError:
3708 fgid = format(int(0), 'x').lower();
3709 try:
3710 import pwd;
3711 try:
3712 userinfo = pwd.getpwuid(os.getuid());
3713 funame = userinfo.pw_name;
3714 except KeyError:
3715 funame = "";
3716 except AttributeError:
3717 funame = "";
3718 except ImportError:
3719 funame = "";
3720 fgname = "";
3721 try:
3722 import grp;
3723 try:
3724 groupinfo = grp.getgrgid(os.getgid());
3725 fgname = groupinfo.gr_name;
3726 except KeyError:
3727 fgname = "";
3728 except AttributeError:
3729 fgname = "";
3730 except ImportError:
3731 fgname = "";
3732 fcontents = BytesIO();
3733 if(ftype==0):
3734 fcontents.write(rarfp.read(member.filename));
3735 if(not compresswholefile):
3736 fcontents.seek(0, 2);
3737 ucfsize = fcontents.tell();
3738 fcontents.seek(0, 0);
3739 if(compression=="auto"):
3740 ilsize = len(compressionlistalt);
3741 ilmin = 0;
3742 ilcsize = [];
3743 while(ilmin < ilsize):
3744 cfcontents = BytesIO();
3745 shutil.copyfileobj(fcontents, cfcontents);
3746 fcontents.seek(0, 0);
3747 cfcontents.seek(0, 0);
3748 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3749 if(cfcontents):
3750 cfcontents.seek(0, 2);
3751 ilcsize.append(cfcontents.tell());
3752 cfcontents.close();
3753 else:
3754 try:
3755 ilcsize.append(sys.maxint);
3756 except AttributeError:
3757 ilcsize.append(sys.maxsize);
3758 ilmin = ilmin + 1;
3759 ilcmin = ilcsize.index(min(ilcsize));
3760 compression = compressionlistalt[ilcmin];
3761 fcontents.seek(0, 0);
3762 cfcontents = BytesIO();
3763 shutil.copyfileobj(fcontents, cfcontents);
3764 cfcontents.seek(0, 0);
3765 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3766 cfcontents.seek(0, 2);
3767 cfsize = cfcontents.tell();
3768 if(ucfsize > cfsize):
3769 fcsize = format(int(cfsize), 'x').lower();
3770 fcompression = compression;
3771 fcontents.close();
3772 fcontents = cfcontents;
3773 if(fcompression=="none"):
3774 fcompression = "";
3775 fcontents.seek(0, 0);
3776 ftypehex = format(ftype, 'x').lower();
3777 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3778 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3779 fcontents.close();
3780 if(numfiles>0):
3781 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3782 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3783 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3784 try:
3785 catfp.flush();
3786 os.fsync(catfp.fileno());
3787 except io.UnsupportedOperation:
3788 pass;
3789 except AttributeError:
3790 pass;
3791 except OSError as e:
3792 pass;
3793 if(outfile=="-"):
3794 catfp.seek(0, 0)
3795 if(hasattr(sys.stdout, "buffer")):
3796 shutil.copyfileobj(catfp, sys.stdout.buffer);
3797 else:
3798 shutil.copyfileobj(catfp, sys.stdout);
3799 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3800 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3801 catfp.seek(0, 0);
3802 upload_file_to_internet_file(catfp, outfile);
3803 if(returnfp):
3804 catfp.seek(0, 0)
3805 return catfp
3806 else:
3807 catfp.close()
3808 return True;
3810 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3812 if(not py7zr_support):
3813 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3814 return False
3816 if(py7zr_support):
3817 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3818 formatspecs = FormatSpecsListToDict(formatspecs);
3819 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3820 outfile = RemoveWindowsPath(outfile);
3821 checksumtype = checksumtype.lower();
3822 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3823 checksumtype="crc32";
3824 if(checksumtype=="none"):
3825 checksumtype = "";
3826 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3827 compression = "auto";
3828 if(compression not in compressionlist and compression is None):
3829 compression = "auto";
3830 if(verbose):
3831 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3832 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3833 if(os.path.exists(outfile)):
3834 try:
3835 os.unlink(outfile);
3836 except OSError as e:
3837 pass;
3838 if(outfile=="-"):
3839 verbose = False;
3840 catfp = BytesIO();
3841 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3842 catfp = outfile;
3843 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3844 catfp = BytesIO();
3845 else:
3846 fbasename = os.path.splitext(outfile)[0];
3847 fextname = os.path.splitext(outfile)[1];
3848 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3849 catver = formatspecs['format_ver'];
3850 fileheaderver = str(int(catver.replace(".", "")));
3851 curinode = 0;
3852 curfid = 0;
3853 inodelist = [];
3854 inodetofile = {};
3855 filetoinode = {};
3856 inodetocatinode = {};
3857 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3858 return False;
3859 szpfp = py7zr.SevenZipFile(infile, mode="r");
3860 file_content = szpfp.readall();
3861 #sztest = szpfp.testzip();
3862 sztestalt = szpfp.test();
3863 if(sztestalt):
3864 VerbosePrintOut("Bad file found!");
3865 numfiles = int(len(szpfp.list()));
3866 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3867 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3868 if(re.findall(r"^[.|/]", member.filename)):
3869 fname = member.filename;
3870 else:
3871 fname = "./"+member.filename;
3872 if(verbose):
3873 VerbosePrintOut(fname);
3874 if(not member.is_directory):
3875 fpremode = int(stat.S_IFREG + 438);
3876 elif(member.is_directory):
3877 fpremode = int(stat.S_IFDIR + 511);
3878 fwinattributes = format(int(0), 'x').lower();
3879 fcompression = "";
3880 fcsize = format(int(0), 'x').lower();
3881 flinkcount = 0;
3882 ftype = 0;
3883 if(member.is_directory):
3884 ftype = 5;
3885 else:
3886 ftype = 0;
3887 flinkname = "";
3888 fcurfid = format(int(curfid), 'x').lower();
3889 fcurinode = format(int(curfid), 'x').lower();
3890 curfid = curfid + 1;
3891 fdev_minor = format(int(0), 'x').lower();
3892 fdev_major = format(int(0), 'x').lower();
3893 frdev_minor = format(int(0), 'x').lower();
3894 frdev_major = format(int(0), 'x').lower();
3895 if(ftype==5):
3896 fsize = format(int("0"), 'x').lower();
3897 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3898 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3899 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3900 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3901 if(member.is_directory):
3902 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3903 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3904 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3905 else:
3906 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3907 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3908 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3909 try:
3910 fuid = format(int(os.getuid()), 'x').lower();
3911 except AttributeError:
3912 fuid = format(int(0), 'x').lower();
3913 except KeyError:
3914 fuid = format(int(0), 'x').lower();
3915 try:
3916 fgid = format(int(os.getgid()), 'x').lower();
3917 except AttributeError:
3918 fgid = format(int(0), 'x').lower();
3919 except KeyError:
3920 fgid = format(int(0), 'x').lower();
3921 try:
3922 import pwd;
3923 try:
3924 userinfo = pwd.getpwuid(os.getuid());
3925 funame = userinfo.pw_name;
3926 except KeyError:
3927 funame = "";
3928 except AttributeError:
3929 funame = "";
3930 except ImportError:
3931 funame = "";
3932 fgname = "";
3933 try:
3934 import grp;
3935 try:
3936 groupinfo = grp.getgrgid(os.getgid());
3937 fgname = groupinfo.gr_name;
3938 except KeyError:
3939 fgname = "";
3940 except AttributeError:
3941 fgname = "";
3942 except ImportError:
3943 fgname = "";
3944 fcontents = BytesIO();
3945 if(ftype==0):
3946 fcontents.write(file_content[member.filename].read());
3947 fsize = format(fcontents.tell(), 'x').lower();
3948 file_content[member.filename].close();
3949 if(not compresswholefile):
3950 fcontents.seek(0, 2);
3951 ucfsize = fcontents.tell();
3952 fcontents.seek(0, 0);
3953 if(compression=="auto"):
3954 ilsize = len(compressionlistalt);
3955 ilmin = 0;
3956 ilcsize = [];
3957 while(ilmin < ilsize):
3958 cfcontents = BytesIO();
3959 shutil.copyfileobj(fcontents, cfcontents);
3960 fcontents.seek(0, 0);
3961 cfcontents.seek(0, 0);
3962 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3963 if(cfcontents):
3964 cfcontents.seek(0, 2);
3965 ilcsize.append(cfcontents.tell());
3966 cfcontents.close();
3967 else:
3968 try:
3969 ilcsize.append(sys.maxint);
3970 except AttributeError:
3971 ilcsize.append(sys.maxsize);
3972 ilmin = ilmin + 1;
3973 ilcmin = ilcsize.index(min(ilcsize));
3974 compression = compressionlistalt[ilcmin];
3975 fcontents.seek(0, 0);
3976 cfcontents = BytesIO();
3977 shutil.copyfileobj(fcontents, cfcontents);
3978 cfcontents.seek(0, 0);
3979 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3980 cfcontents.seek(0, 2);
3981 cfsize = cfcontents.tell();
3982 if(ucfsize > cfsize):
3983 fcsize = format(int(cfsize), 'x').lower();
3984 fcompression = compression;
3985 fcontents.close();
3986 fcontents = cfcontents;
3987 if(fcompression=="none"):
3988 fcompression = "";
3989 fcontents.seek(0, 0);
3990 ftypehex = format(ftype, 'x').lower();
3991 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3992 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3993 fcontents.close();
3994 if(numfiles>0):
3995 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3996 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3997 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3998 try:
3999 catfp.flush();
4000 os.fsync(catfp.fileno());
4001 except io.UnsupportedOperation:
4002 pass;
4003 except AttributeError:
4004 pass;
4005 except OSError as e:
4006 pass;
4007 if(outfile=="-"):
4008 catfp.seek(0, 0);
4009 if(hasattr(sys.stdout, "buffer")):
4010 shutil.copyfileobj(catfp, sys.stdout.buffer);
4011 else:
4012 shutil.copyfileobj(catfp, sys.stdout);
4013 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4014 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4015 catfp.seek(0, 0);
4016 upload_file_to_internet_file(catfp, outfile);
4017 if(returnfp):
4018 catfp.seek(0, 0)
4019 return catfp
4020 else:
4021 catfp.close()
4022 return True;
4024 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4026 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4027 formatspecs = FormatSpecsListToDict(formatspecs);
4028 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4029 if(verbose):
4030 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4031 if(checkcompressfile=="tarfile"):
4032 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4033 elif(checkcompressfile=="zipfile"):
4034 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4035 elif(checkcompressfile=="catfile"):
4036 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4037 elif(rarfile_support and checkcompressfile=="rarfile"):
4038 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4039 elif(py7zr_support and checkcompressfile=="7zipfile"):
4040 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4041 else:
4042 return False;
4043 return False;
4045 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4047 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4048 formatspecs = FormatSpecsListToDict(formatspecs);
4049 if(hasattr(infile, "read") or hasattr(infile, "write")):
4050 catfp = infile;
4051 catfp.seek(0, 0);
4052 catfp = UncompressArchiveFile(catfp, formatspecs);
4053 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4054 if(checkcompressfile=="tarfile"):
4055 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4056 if(checkcompressfile=="zipfile"):
4057 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4058 if(rarfile_support and checkcompressfile=="rarfile"):
4059 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4060 if(py7zr_support and checkcompressfile=="7zipfile"):
4061 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4062 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4063 return False;
4064 if(not catfp):
4065 return False;
4066 catfp.seek(0, 0);
4067 elif(infile=="-"):
4068 catfp = BytesIO();
4069 if(hasattr(sys.stdin, "buffer")):
4070 shutil.copyfileobj(sys.stdin.buffer, catfp);
4071 else:
4072 shutil.copyfileobj(sys.stdin, catfp);
4073 catfp.seek(0, 0);
4074 catfp = UncompressArchiveFile(catfp, formatspecs);
4075 if(not catfp):
4076 return False;
4077 catfp.seek(0, 0);
4078 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4079 catfp = download_file_from_internet_file(infile);
4080 catfp.seek(0, 0);
4081 catfp = UncompressArchiveFile(catfp, formatspecs);
4082 if(not catfp):
4083 return False;
4084 catfp.seek(0, 0);
4085 else:
4086 infile = RemoveWindowsPath(infile);
4087 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4088 if(checkcompressfile=="tarfile"):
4089 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4090 if(checkcompressfile=="zipfile"):
4091 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4092 if(rarfile_support and checkcompressfile=="rarfile"):
4093 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4094 if(py7zr_support and checkcompressfile=="7zipfile"):
4095 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4096 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4097 return False;
4098 compresscheck = CheckCompressionType(infile, formatspecs, True);
4099 if(not compresscheck):
4100 fextname = os.path.splitext(infile)[1];
4101 if(fextname==".gz"):
4102 compresscheck = "gzip";
4103 elif(fextname==".bz2"):
4104 compresscheck = "bzip2";
4105 elif(fextname==".zst"):
4106 compresscheck = "zstd";
4107 elif(fextname==".lz4" or fextname==".clz4"):
4108 compresscheck = "lz4";
4109 elif(fextname==".lzo" or fextname==".lzop"):
4110 compresscheck = "lzo";
4111 elif(fextname==".lzma" or fextname==".xz"):
4112 compresscheck = "lzma";
4113 else:
4114 return False;
4115 if(not compresscheck):
4116 return False;
4117 catfp = UncompressFile(infile, formatspecs, "rb");
4119 try:
4120 catfp.seek(0, 2);
4121 except OSError:
4122 SeekToEndOfFile(catfp);
4123 except ValueError:
4124 SeekToEndOfFile(catfp);
4125 CatSize = catfp.tell();
4126 CatSizeEnd = CatSize;
4128 try:
4129 catfp.seek(0, 0);
4130 except OSError:
4131 return False;
4132 except ValueError:
4133 return False;
4134 curloc = catfp.tell();
4135 if(curloc>0):
4136 catfp.seek(0, 0);
4137 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4138 if(curloc>0):
4139 catfp.seek(curloc, 0);
4140 catstring = catheader[0];
4141 catversion = re.findall(r"([\d]+)", catstring);
4142 fprenumfiles = catheader[1];
4143 fnumfiles = int(fprenumfiles, 16);
4144 fprechecksumtype = catheader[2];
4145 fprechecksum = catheader[3];
4146 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4147 fnumfileshex = format(int(fnumfiles), 'x').lower();
4148 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4149 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4150 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4151 fheadtell = len(fileheader);
4152 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4153 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4154 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4155 return False;
4156 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4157 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4158 if(seekto>=fnumfiles):
4159 seekto = fnumfiles - 1;
4160 if(seekto<0):
4161 seekto = 0;
4162 if(seekto>=0):
4163 il = -1;
4164 while(il < seekto):
4165 prefhstart = catfp.tell();
4166 if(formatspecs['new_style']):
4167 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4168 else:
4169 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4170 if(len(preheaderdata)==0):
4171 break;
4172 prefheadsize = int(preheaderdata[0], 16);
4173 prefnumfields = int(preheaderdata[1], 16);
4174 preftype = int(preheaderdata[2], 16);
4175 if(re.findall(r"^[.|/]", preheaderdata[3])):
4176 prefname = preheaderdata[3];
4177 else:
4178 prefname = "./"+preheaderdata[3];
4179 prefbasedir = os.path.dirname(prefname);
4180 preflinkname = preheaderdata[4];
4181 prefsize = int(preheaderdata[5], 16);
4182 prefatime = int(preheaderdata[6], 16);
4183 prefmtime = int(preheaderdata[7], 16);
4184 prefctime = int(preheaderdata[8], 16);
4185 prefbtime = int(preheaderdata[9], 16);
4186 prefmode = int(preheaderdata[10], 16);
4187 prefchmode = stat.S_IMODE(prefmode);
4188 preftypemod = stat.S_IFMT(prefmode);
4189 prefwinattributes = int(preheaderdata[11], 16);
4190 prefcompression = preheaderdata[12];
4191 prefcsize = int(preheaderdata[13], 16);
4192 prefuid = int(preheaderdata[14], 16);
4193 prefuname = preheaderdata[15];
4194 prefgid = int(preheaderdata[16], 16);
4195 prefgname = preheaderdata[17];
4196 fid = int(preheaderdata[18], 16);
4197 finode = int(preheaderdata[19], 16);
4198 flinkcount = int(preheaderdata[20], 16);
4199 prefdev_minor = int(preheaderdata[21], 16);
4200 prefdev_major = int(preheaderdata[22], 16);
4201 prefrdev_minor = int(preheaderdata[23], 16);
4202 prefrdev_major = int(preheaderdata[24], 16);
4203 prefseeknextfile = preheaderdata[25];
4204 prefextrasize = int(preheaderdata[26], 16);
4205 prefextrafields = int(preheaderdata[27], 16);
4206 extrafieldslist = [];
4207 extrastart = 28;
4208 extraend = extrastart + prefextrafields;
4209 extrafieldslist = [];
4210 if(extrastart<extraend):
4211 extrafieldslist.append(preheaderdata[extrastart]);
4212 extrastart = extrastart + 1;
4213 prefcs = preheaderdata[extrastart + 2].lower();
4214 prefccs = preheaderdata[extrastart + 3].lower();
4215 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4216 if(prefcs!=prenewfcs and not skipchecksum):
4217 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4218 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4219 return False;
4220 valid_archive = False;
4221 invalid_archive = True;
4222 prefhend = catfp.tell() - 1;
4223 prefcontentstart = catfp.tell();
4224 prefcontents = "";
4225 pyhascontents = False;
4226 if(prefsize>0):
4227 if(prefcompression):
4228 prefcontents = catfp.read(prefsize);
4229 else:
4230 prefcontents = catfp.read(prefcsize);
4231 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4232 pyhascontents = True;
4233 if(prefccs!=prenewfccs and not skipchecksum):
4234 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4235 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4236 return False;
4237 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4238 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4239 if(abs(fseeknextasnum)==0):
4240 pass;
4241 catfp.seek(fseeknextasnum, 1);
4242 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4243 fseeknextasnum = int(prefseeknextfile);
4244 if(abs(fseeknextasnum)==0):
4245 pass;
4246 catfp.seek(fseeknextasnum, 1);
4247 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4248 fseeknextasnum = int(prefseeknextfile);
4249 if(abs(fseeknextasnum)==0):
4250 pass;
4251 catfp.seek(fseeknextasnum, 0);
4252 else:
4253 return False;
4254 il = il + 1;
4255 catfp.seek(seekstart, 0);
4256 fileidnum = il;
4257 catfheadsize = int(preheaderdata[0], 16);
4258 catfnumfields = int(preheaderdata[1], 16);
4259 catftype = int(preheaderdata[2], 16);
4260 if(re.findall(r"^[.|/]", preheaderdata[3])):
4261 catfname = preheaderdata[3];
4262 else:
4263 catfname = "./"+preheaderdata[3];
4264 catflinkname = preheaderdata[4];
4265 catfsize = int(preheaderdata[5], 16);
4266 catfbasedir = os.path.dirname(catfname);
4267 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4268 if(returnfp):
4269 catlist.update({'catfp': catfp});
4270 else:
4271 catfp.close();
4272 return catlist;
4274 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4276 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4277 formatspecs = FormatSpecsListToDict(formatspecs);
4278 if(hasattr(infile, "read") or hasattr(infile, "write")):
4279 catfp = infile;
4280 catfp.seek(0, 0);
4281 catfp = UncompressArchiveFile(catfp, formatspecs);
4282 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4283 if(checkcompressfile=="tarfile"):
4284 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4285 if(checkcompressfile=="zipfile"):
4286 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4287 if(rarfile_support and checkcompressfile=="rarfile"):
4288 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4289 if(py7zr_support and checkcompressfile=="7zipfile"):
4290 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4291 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4292 return False;
4293 if(not catfp):
4294 return False;
4295 catfp.seek(0, 0);
4296 elif(infile=="-"):
4297 catfp = BytesIO();
4298 if(hasattr(sys.stdin, "buffer")):
4299 shutil.copyfileobj(sys.stdin.buffer, catfp);
4300 else:
4301 shutil.copyfileobj(sys.stdin, catfp);
4302 catfp.seek(0, 0);
4303 catfp = UncompressArchiveFile(catfp, formatspecs);
4304 if(not catfp):
4305 return False;
4306 catfp.seek(0, 0);
4307 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4308 catfp = download_file_from_internet_file(infile);
4309 catfp = UncompressArchiveFile(catfp, formatspecs);
4310 catfp.seek(0, 0);
4311 if(not catfp):
4312 return False;
4313 catfp.seek(0, 0);
4314 else:
4315 infile = RemoveWindowsPath(infile);
4316 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4317 if(checkcompressfile=="tarfile"):
4318 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4319 if(checkcompressfile=="zipfile"):
4320 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4321 if(rarfile_support and checkcompressfile=="rarfile"):
4322 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4323 if(py7zr_support and checkcompressfile=="7zipfile"):
4324 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4325 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4326 return False;
4327 compresscheck = CheckCompressionType(infile, formatspecs, True);
4328 if(not compresscheck):
4329 fextname = os.path.splitext(infile)[1];
4330 if(fextname==".gz"):
4331 compresscheck = "gzip";
4332 elif(fextname==".bz2"):
4333 compresscheck = "bzip2";
4334 elif(fextname==".zst"):
4335 compresscheck = "zstd";
4336 elif(fextname==".lz4" or fextname==".clz4"):
4337 compresscheck = "lz4";
4338 elif(fextname==".lzo" or fextname==".lzop"):
4339 compresscheck = "lzo";
4340 elif(fextname==".lzma" or fextname==".xz"):
4341 compresscheck = "lzma";
4342 else:
4343 return False;
4344 if(not compresscheck):
4345 return False;
4346 catfp = UncompressFile(infile, formatspecs, "rb");
4348 try:
4349 catfp.seek(0, 2);
4350 except OSError:
4351 SeekToEndOfFile(catfp);
4352 except ValueError:
4353 SeekToEndOfFile(catfp);
4354 CatSize = catfp.tell();
4355 CatSizeEnd = CatSize;
4357 try:
4358 catfp.seek(0, 0);
4359 except OSError:
4360 return False;
4361 except ValueError:
4362 return False;
4363 curloc = catfp.tell();
4364 if(curloc>0):
4365 catfp.seek(0, 0);
4366 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4367 if(curloc>0):
4368 catfp.seek(curloc, 0);
4369 catstring = catheader[0];
4370 catversion = re.findall(r"([\d]+)", catstring);
4371 fprenumfiles = catheader[1];
4372 fnumfiles = int(fprenumfiles, 16);
4373 fprechecksumtype = catheader[2];
4374 fprechecksum = catheader[3];
4375 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4376 fnumfileshex = format(int(fnumfiles), 'x').lower();
4377 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4378 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4379 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4380 fheadtell = len(fileheader);
4381 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4382 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4383 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4384 return False;
4385 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4386 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4387 seekto = fnumfiles - 1
4388 filefound = False;
4389 if(seekto>=0):
4390 il = -1;
4391 while(il < seekto):
4392 prefhstart = catfp.tell();
4393 if(formatspecs['new_style']):
4394 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4395 else:
4396 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4397 if(len(preheaderdata)==0):
4398 break;
4399 prefheadsize = int(preheaderdata[0], 16);
4400 prefnumfields = int(preheaderdata[1], 16);
4401 preftype = int(preheaderdata[2], 16);
4402 if(re.findall(r"^[.|/]", preheaderdata[3])):
4403 prefname = preheaderdata[3];
4404 else:
4405 prefname = "./"+preheaderdata[3];
4406 prefbasedir = os.path.dirname(prefname);
4407 preflinkname = preheaderdata[4];
4408 prefsize = int(preheaderdata[5], 16);
4409 prefatime = int(preheaderdata[6], 16);
4410 prefmtime = int(preheaderdata[7], 16);
4411 prefctime = int(preheaderdata[8], 16);
4412 prefbtime = int(preheaderdata[9], 16);
4413 prefmode = int(preheaderdata[10], 16);
4414 prefchmode = stat.S_IMODE(prefmode);
4415 preftypemod = stat.S_IFMT(prefmode);
4416 prefwinattributes = int(preheaderdata[11], 16);
4417 prefcompression = preheaderdata[12];
4418 prefcsize = int(preheaderdata[13], 16);
4419 prefuid = int(preheaderdata[14], 16);
4420 prefuname = preheaderdata[15];
4421 prefgid = int(preheaderdata[16], 16);
4422 prefgname = preheaderdata[17];
4423 fid = int(preheaderdata[18], 16);
4424 finode = int(preheaderdata[19], 16);
4425 flinkcount = int(preheaderdata[20], 16);
4426 prefdev_minor = int(preheaderdata[21], 16);
4427 prefdev_major = int(preheaderdata[22], 16);
4428 prefrdev_minor = int(preheaderdata[23], 16);
4429 prefrdev_major = int(preheaderdata[24], 16);
4430 prefseeknextfile = preheaderdata[25];
4431 prefextrasize = int(preheaderdata[26], 16);
4432 prefextrafields = int(preheaderdata[27], 16);
4433 extrafieldslist = [];
4434 extrastart = 28;
4435 extraend = extrastart + prefextrafields;
4436 extrafieldslist = [];
4437 if(extrastart<extraend):
4438 extrafieldslist.append(preheaderdata[extrastart]);
4439 extrastart = extrastart + 1;
4440 prefcs = preheaderdata[extrastart + 2].lower();
4441 prefccs = preheaderdata[extrastart + 3].lower();
4442 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4443 if(prefcs!=prenewfcs and not skipchecksum):
4444 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4445 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4446 return False;
4447 valid_archive = False;
4448 invalid_archive = True;
4449 prefhend = catfp.tell() - 1;
4450 prefcontentstart = catfp.tell();
4451 prefcontents = "";
4452 pyhascontents = False;
4453 if(prefsize>0):
4454 if(prefcompression):
4455 prefcontents = catfp.read(prefsize);
4456 else:
4457 prefcontents = catfp.read(prefcsize);
4458 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4459 pyhascontents = True;
4460 if(prefccs!=prenewfccs and not skipchecksum):
4461 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4462 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4463 return False;
4464 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4465 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4466 if(abs(fseeknextasnum)==0):
4467 pass;
4468 catfp.seek(fseeknextasnum, 1);
4469 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4470 fseeknextasnum = int(prefseeknextfile);
4471 if(abs(fseeknextasnum)==0):
4472 pass;
4473 catfp.seek(fseeknextasnum, 1);
4474 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4475 fseeknextasnum = int(prefseeknextfile);
4476 if(abs(fseeknextasnum)==0):
4477 pass;
4478 catfp.seek(fseeknextasnum, 0);
4479 else:
4480 return False;
4481 il = il + 1;
4482 filefound = False;
4483 prefname = preheaderdata[2];
4484 if(re.findall(r"^[.|/]", preheaderdata[2])):
4485 prefname = preheaderdata[2];
4486 else:
4487 prefname = "./"+preheaderdata[2];
4488 if(prefname==seekfile):
4489 filefound = True;
4490 break;
4491 catfp.seek(seekstart, 0);
4492 fileidnum = il;
4493 catfheadsize = int(preheaderdata[0], 16);
4494 catfnumfields = int(preheaderdata[1], 16);
4495 catftype = int(preheaderdata[2], 16);
4496 if(re.findall(r"^[.|/]", preheaderdata[3])):
4497 catfname = preheaderdata[3];
4498 else:
4499 catfname = "./"+preheaderdata[3];
4500 catflinkname = preheaderdata[4];
4501 catfsize = int(preheaderdata[5], 16);
4502 catfbasedir = os.path.dirname(catfname);
4503 if(filefound):
4504 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4505 else:
4506 return False;
4507 if(returnfp):
4508 catlist.update({'catfp': catfp});
4509 else:
4510 catfp.close();
4511 return catlist;
4513 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4515 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4516 formatspecs = FormatSpecsListToDict(formatspecs);
4517 if(verbose):
4518 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4519 if(hasattr(infile, "read") or hasattr(infile, "write")):
4520 catfp = infile;
4521 catfp.seek(0, 0);
4522 catfp = UncompressArchiveFile(catfp, formatspecs);
4523 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4524 if(checkcompressfile=="tarfile"):
4525 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4526 if(checkcompressfile=="zipfile"):
4527 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4528 if(rarfile_support and checkcompressfile=="rarfile"):
4529 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4530 if(py7zr_support and checkcompressfile=="7zipfile"):
4531 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4532 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4533 return False;
4534 if(not catfp):
4535 return False;
4536 catfp.seek(0, 0);
4537 elif(infile=="-"):
4538 catfp = BytesIO();
4539 if(hasattr(sys.stdin, "buffer")):
4540 shutil.copyfileobj(sys.stdin.buffer, catfp);
4541 else:
4542 shutil.copyfileobj(sys.stdin, catfp);
4543 catfp.seek(0, 0);
4544 catfp = UncompressArchiveFile(catfp, formatspecs);
4545 if(not catfp):
4546 return False;
4547 catfp.seek(0, 0);
4548 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4549 catfp = download_file_from_internet_file(infile);
4550 catfp = UncompressArchiveFile(catfp, formatspecs);
4551 catfp.seek(0, 0);
4552 if(not catfp):
4553 return False;
4554 catfp.seek(0, 0);
4555 else:
4556 infile = RemoveWindowsPath(infile);
4557 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4558 if(checkcompressfile=="tarfile"):
4559 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4560 if(checkcompressfile=="zipfile"):
4561 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4562 if(rarfile_support and checkcompressfile=="rarfile"):
4563 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4564 if(py7zr_support and checkcompressfile=="7zipfile"):
4565 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4566 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4567 return False;
4568 compresscheck = CheckCompressionType(infile, formatspecs, True);
4569 if(not compresscheck):
4570 fextname = os.path.splitext(infile)[1];
4571 if(fextname==".gz"):
4572 compresscheck = "gzip";
4573 elif(fextname==".bz2"):
4574 compresscheck = "bzip2";
4575 elif(fextname==".zst"):
4576 compresscheck = "zstd";
4577 elif(fextname==".lz4" or fextname==".clz4"):
4578 compresscheck = "lz4";
4579 elif(fextname==".lzo" or fextname==".lzop"):
4580 compresscheck = "lzo";
4581 elif(fextname==".lzma" or fextname==".xz"):
4582 compresscheck = "lzma";
4583 else:
4584 return False;
4585 if(not compresscheck):
4586 return False;
4587 catfp = UncompressFile(infile, formatspecs, "rb");
4589 try:
4590 catfp.seek(0, 2);
4591 except OSError:
4592 SeekToEndOfFile(catfp);
4593 except ValueError:
4594 SeekToEndOfFile(catfp);
4595 CatSize = catfp.tell();
4596 CatSizeEnd = CatSize;
4598 try:
4599 catfp.seek(0, 0);
4600 except OSError:
4601 return False;
4602 except ValueError:
4603 return False;
4604 curloc = catfp.tell();
4605 if(curloc>0):
4606 catfp.seek(0, 0);
4607 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4608 if(curloc>0):
4609 catfp.seek(curloc, 0);
4610 catstring = catheader[0];
4611 catversion = re.findall(r"([\d]+)", catstring);
4612 fprenumfiles = catheader[1];
4613 fnumfiles = int(fprenumfiles, 16);
4614 fprechecksumtype = catheader[2];
4615 fprechecksum = catheader[3];
4616 il = 0;
4617 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4618 fnumfileshex = format(int(fnumfiles), 'x').lower();
4619 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4620 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4621 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4622 valid_archive = True;
4623 invalid_archive = False;
4624 if(verbose):
4625 VerbosePrintOut(infile);
4626 VerbosePrintOut("Number of Records " + str(fnumfiles));
4627 if(fprechecksum==catfileheadercshex):
4628 if(verbose):
4629 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4630 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4631 else:
4632 if(verbose):
4633 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4634 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4635 valid_archive = False;
4636 invalid_archive = True;
4637 if(verbose):
4638 VerbosePrintOut("");
4639 while(il<fnumfiles):
4640 catfhstart = catfp.tell();
4641 if(formatspecs['new_style']):
4642 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4643 else:
4644 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4645 if(len(catheaderdata)==0):
4646 break;
4647 catfheadsize = int(catheaderdata[0], 16);
4648 catfnumfields = int(catheaderdata[1], 16);
4649 catftype = int(catheaderdata[2], 16);
4650 if(re.findall(r"^[.|/]", catheaderdata[3])):
4651 catfname = catheaderdata[3];
4652 else:
4653 catfname = "./"+catheaderdata[3];
4654 catfbasedir = os.path.dirname(catfname);
4655 catflinkname = catheaderdata[4];
4656 catfsize = int(catheaderdata[5], 16);
4657 catfatime = int(catheaderdata[6], 16);
4658 catfmtime = int(catheaderdata[7], 16);
4659 catfctime = int(catheaderdata[8], 16);
4660 catfbtime = int(catheaderdata[9], 16);
4661 catfmode = int(catheaderdata[10], 16);
4662 catfchmode = stat.S_IMODE(catfmode);
4663 catftypemod = stat.S_IFMT(catfmode);
4664 catfwinattributes = int(catheaderdata[11], 16);
4665 catfcompression = catheaderdata[12];
4666 catfcsize = int(catheaderdata[13], 16);
4667 catfuid = int(catheaderdata[14], 16);
4668 catfuname = catheaderdata[15];
4669 catfgid = int(catheaderdata[16], 16);
4670 catfgname = catheaderdata[17];
4671 fid = int(catheaderdata[18], 16);
4672 finode = int(catheaderdata[19], 16);
4673 flinkcount = int(catheaderdata[20], 16);
4674 catfdev_minor = int(catheaderdata[21], 16);
4675 catfdev_major = int(catheaderdata[22], 16);
4676 catfrdev_minor = int(catheaderdata[23], 16);
4677 catfrdev_major = int(catheaderdata[24], 16);
4678 catfseeknextfile = catheaderdata[25];
4679 catfextrasize = int(catheaderdata[26], 16);
4680 catfextrafields = int(catheaderdata[27], 16);
4681 extrafieldslist = [];
4682 extrastart = 28;
4683 extraend = extrastart + catfextrafields;
4684 extrafieldslist = [];
4685 if(extrastart<extraend):
4686 extrafieldslist.append(catheaderdata[extrastart]);
4687 extrastart = extrastart + 1;
4688 catfcs = catheaderdata[extrastart + 2].lower();
4689 catfccs = catheaderdata[extrastart + 3].lower();
4690 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4691 if(verbose):
4692 VerbosePrintOut(catfname);
4693 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4694 if(catfcs==catnewfcs):
4695 if(verbose):
4696 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4697 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4698 else:
4699 if(verbose):
4700 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4701 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4702 valid_archive = False;
4703 invalid_archive = True;
4704 catfhend = catfp.tell() - 1;
4705 catfcontentstart = catfp.tell();
4706 catfcontents = "";
4707 pyhascontents = False;
4708 if(catfsize>0):
4709 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4710 catfcontents = catfp.read(catfsize);
4711 else:
4712 catfcontents = catfp.read(catfcsize);
4713 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4714 pyhascontents = True;
4715 if(catfccs==catnewfccs):
4716 if(verbose):
4717 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4718 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4719 else:
4720 if(verbose):
4721 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4722 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4723 valid_archive = False;
4724 invalid_archive = True;
4725 if(verbose):
4726 VerbosePrintOut("");
4727 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
4728 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
4729 if(abs(fseeknextasnum)==0):
4730 pass;
4731 catfp.seek(fseeknextasnum, 1);
4732 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
4733 fseeknextasnum = int(catfseeknextfile);
4734 if(abs(fseeknextasnum)==0):
4735 pass;
4736 catfp.seek(fseeknextasnum, 1);
4737 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
4738 fseeknextasnum = int(catfseeknextfile);
4739 if(abs(fseeknextasnum)==0):
4740 pass;
4741 catfp.seek(fseeknextasnum, 0);
4742 else:
4743 return False;
4744 il = il + 1;
4745 if(valid_archive):
4746 if(returnfp):
4747 return catfp;
4748 else:
4749 catfp.close();
4750 return True;
4751 else:
4752 catfp.close();
4753 return False;
4755 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4757 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4758 formatspecs = FormatSpecsListToDict(formatspecs);
4759 if(hasattr(infile, "read") or hasattr(infile, "write")):
4760 catfp = infile;
4761 catfp.seek(0, 0);
4762 catfp = UncompressArchiveFile(catfp, formatspecs);
4763 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4764 if(checkcompressfile=="tarfile"):
4765 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4766 if(checkcompressfile=="zipfile"):
4767 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4768 if(rarfile_support and checkcompressfile=="rarfile"):
4769 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4770 if(py7zr_support and checkcompressfile=="7zipfile"):
4771 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4772 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4773 return False;
4774 if(not catfp):
4775 return False;
4776 catfp.seek(0, 0);
4777 elif(infile=="-"):
4778 catfp = BytesIO();
4779 if(hasattr(sys.stdin, "buffer")):
4780 shutil.copyfileobj(sys.stdin.buffer, catfp);
4781 else:
4782 shutil.copyfileobj(sys.stdin, catfp);
4783 catfp.seek(0, 0);
4784 catfp = UncompressArchiveFile(catfp, formatspecs);
4785 if(not catfp):
4786 return False;
4787 catfp.seek(0, 0);
4788 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4789 catfp = download_file_from_internet_file(infile);
4790 catfp = UncompressArchiveFile(catfp, formatspecs);
4791 catfp.seek(0, 0);
4792 if(not catfp):
4793 return False;
4794 catfp.seek(0, 0);
4795 else:
4796 infile = RemoveWindowsPath(infile);
4797 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4798 if(checkcompressfile=="tarfile"):
4799 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4800 if(checkcompressfile=="zipfile"):
4801 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4802 if(rarfile_support and checkcompressfile=="rarfile"):
4803 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4804 if(py7zr_support and checkcompressfile=="7zipfile"):
4805 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4806 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4807 return False;
4808 compresscheck = CheckCompressionType(infile, formatspecs, True);
4809 if(not compresscheck):
4810 fextname = os.path.splitext(infile)[1];
4811 if(fextname==".gz"):
4812 compresscheck = "gzip";
4813 elif(fextname==".bz2"):
4814 compresscheck = "bzip2";
4815 elif(fextname==".zst"):
4816 compresscheck = "zstd";
4817 elif(fextname==".lz4" or fextname==".clz4"):
4818 compresscheck = "lz4";
4819 elif(fextname==".lzo" or fextname==".lzop"):
4820 compresscheck = "lzo";
4821 elif(fextname==".lzma" or fextname==".xz"):
4822 compresscheck = "lzma";
4823 else:
4824 return False;
4825 if(not compresscheck):
4826 return False;
4827 catfp = UncompressFile(infile, formatspecs, "rb");
4829 try:
4830 catfp.seek(0, 2);
4831 except OSError:
4832 SeekToEndOfFile(catfp);
4833 except ValueError:
4834 SeekToEndOfFile(catfp);
4835 CatSize = catfp.tell();
4836 CatSizeEnd = CatSize;
4838 try:
4839 catfp.seek(0, 0);
4840 except OSError:
4841 return False;
4842 except ValueError:
4843 return False;
4844 curloc = catfp.tell();
4845 if(curloc>0):
4846 catfp.seek(0, 0);
4847 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4848 if(curloc>0):
4849 catfp.seek(curloc, 0);
4850 catstring = catheader[0];
4851 catversion = re.findall(r"([\d]+)", catstring);
4852 fprenumfiles = catheader[1];
4853 fnumfiles = int(fprenumfiles, 16);
4854 fprechecksumtype = catheader[2];
4855 fprechecksum = catheader[3];
4856 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4857 fnumfileshex = format(int(fnumfiles), 'x').lower();
4858 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4859 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4860 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4861 fheadtell = len(fileheader);
4862 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4863 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4864 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4865 return False;
4866 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4867 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
4868 if(seekstart<0 and seekstart>fnumfiles):
4869 seekstart = 0;
4870 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4871 seekend = fnumfiles;
4872 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4873 seekend = fnumfiles - abs(seekend);
4874 if(seekstart>0):
4875 il = 0;
4876 while(il < seekstart):
4877 prefhstart = catfp.tell();
4878 if(formatspecs['new_style']):
4879 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4880 else:
4881 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4882 if(len(preheaderdata)==0):
4883 break;
4884 prefheadsize = int(preheaderdata[0], 16);
4885 prefnumfields = int(preheaderdata[1], 16);
4886 if(re.findall(r"^[.|/]", preheaderdata[3])):
4887 prefname = preheaderdata[3];
4888 else:
4889 prefname = "./"+preheaderdata[3];
4890 prefsize = int(preheaderdata[5], 16);
4891 prefcompression = preheaderdata[12];
4892 prefcsize = int(preheaderdata[13], 16);
4893 prefseeknextfile = preheaderdata[25];
4894 prefextrasize = int(preheaderdata[26], 16);
4895 prefextrafields = int(preheaderdata[27], 16);
4896 extrafieldslist = [];
4897 extrastart = 28;
4898 extraend = extrastart + prefextrafields;
4899 extrafieldslist = [];
4900 if(extrastart<extraend):
4901 extrafieldslist.append(preheaderdata[extrastart]);
4902 extrastart = extrastart + 1;
4903 prefcs = preheaderdata[extrastart + 2].lower();
4904 prefccs = preheaderdata[extrastart + 3].lower();
4905 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4906 if(prefcs!=prenewfcs and not skipchecksum):
4907 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4908 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4909 return False;
4910 valid_archive = False;
4911 invalid_archive = True;
4912 prefhend = catfp.tell() - 1;
4913 prefcontentstart = catfp.tell();
4914 prefcontents = "";
4915 pyhascontents = False;
4916 if(prefsize>0):
4917 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4918 prefcontents = catfp.read(prefsize);
4919 else:
4920 prefcontents = catfp.read(prefcsize);
4921 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4922 pyhascontents = True;
4923 if(prefccs!=prenewfccs and not skipchecksum):
4924 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4925 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4926 return False;
4927 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4928 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4929 if(abs(fseeknextasnum)==0):
4930 pass;
4931 catfp.seek(fseeknextasnum, 1);
4932 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4933 fseeknextasnum = int(prefseeknextfile);
4934 if(abs(fseeknextasnum)==0):
4935 pass;
4936 catfp.seek(fseeknextasnum, 1);
4937 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4938 fseeknextasnum = int(prefseeknextfile);
4939 if(abs(fseeknextasnum)==0):
4940 pass;
4941 catfp.seek(fseeknextasnum, 0);
4942 else:
4943 return False;
4944 il = il + 1;
4945 fileidnum = seekstart;
4946 realidnum = 0;
4947 while(fileidnum<seekend):
4948 catfhstart = catfp.tell();
4949 if(formatspecs['new_style']):
4950 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4951 else:
4952 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4953 if(len(catheaderdata)==0):
4954 break;
4955 catfheadsize = int(catheaderdata[0], 16);
4956 catfnumfields = int(catheaderdata[1], 16);
4957 catftype = int(catheaderdata[2], 16);
4958 if(re.findall(r"^[.|/]", catheaderdata[3])):
4959 catfname = catheaderdata[3];
4960 else:
4961 catfname = "./"+catheaderdata[3];
4962 catfbasedir = os.path.dirname(catfname);
4963 catflinkname = catheaderdata[4];
4964 catfsize = int(catheaderdata[5], 16);
4965 catfatime = int(catheaderdata[6], 16);
4966 catfmtime = int(catheaderdata[7], 16);
4967 catfctime = int(catheaderdata[8], 16);
4968 catfbtime = int(catheaderdata[9], 16);
4969 catfmode = int(catheaderdata[10], 16);
4970 catfchmode = stat.S_IMODE(catfmode);
4971 catftypemod = stat.S_IFMT(catfmode);
4972 catfwinattributes = int(catheaderdata[11], 16);
4973 catfcompression = catheaderdata[12];
4974 catfcsize = int(catheaderdata[13], 16);
4975 catfuid = int(catheaderdata[14], 16);
4976 catfuname = catheaderdata[15];
4977 catfgid = int(catheaderdata[16], 16);
4978 catfgname = catheaderdata[17];
4979 catfid = int(catheaderdata[18], 16);
4980 catfinode = int(catheaderdata[19], 16);
4981 catflinkcount = int(catheaderdata[20], 16);
4982 catfdev_minor = int(catheaderdata[21], 16);
4983 catfdev_major = int(catheaderdata[22], 16);
4984 catfrdev_minor = int(catheaderdata[23], 16);
4985 catfrdev_major = int(catheaderdata[24], 16);
4986 catfseeknextfile = catheaderdata[25];
4987 catfextrasize = int(catheaderdata[26], 16);
4988 catfextrafields = int(catheaderdata[27], 16);
4989 extrafieldslist = [];
4990 extrastart = 28;
4991 extraend = extrastart + catfextrafields;
4992 extrafieldslist = [];
4993 if(extrastart<extraend):
4994 extrafieldslist.append(catheaderdata[extrastart]);
4995 extrastart = extrastart + 1;
4996 catfcs = catheaderdata[extrastart + 2].lower();
4997 catfccs = catheaderdata[extrastart + 3].lower();
4998 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4999 if(catfcs!=catnewfcs and not skipchecksum):
5000 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
5001 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5002 return False;
5003 catfhend = catfp.tell() - 1;
5004 catfcontentstart = catfp.tell();
5005 catfcontents = BytesIO();
5006 pyhascontents = False;
5007 if(catfsize>0 and not listonly):
5008 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5009 catfcontents.write(catfp.read(catfsize));
5010 else:
5011 catfcontents.write(catfp.read(catfcsize));
5012 catfcontents.seek(0, 0);
5013 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5014 pyhascontents = True;
5015 if(catfccs!=catnewfccs and skipchecksum):
5016 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5017 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5018 return False;
5019 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5020 pass;
5021 else:
5022 catfcontents.seek(0, 0);
5023 if(uncompress):
5024 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5025 catfcontents.seek(0, 0);
5026 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5027 if(catfsize>0 and listonly):
5028 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5029 catfp.seek(catfsize, 1);
5030 else:
5031 catfp.seek(catfcsize, 1);
5032 pyhascontents = False;
5033 catfcontentend = catfp.tell();
5034 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5035 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5036 if(abs(fseeknextasnum)==0):
5037 pass;
5038 catfp.seek(fseeknextasnum, 1);
5039 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5040 fseeknextasnum = int(catfseeknextfile);
5041 if(abs(fseeknextasnum)==0):
5042 pass;
5043 catfp.seek(fseeknextasnum, 1);
5044 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5045 fseeknextasnum = int(catfseeknextfile);
5046 if(abs(fseeknextasnum)==0):
5047 pass;
5048 catfp.seek(fseeknextasnum, 0);
5049 else:
5050 return False;
5051 catfcontents.seek(0, 0);
5052 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
5053 fileidnum = fileidnum + 1;
5054 realidnum = realidnum + 1;
5055 if(returnfp):
5056 catlist.update({'catfp': catfp});
5057 else:
5058 catfp.close();
5059 return catlist;
5061 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5063 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5064 formatspecs = FormatSpecsListToDict(formatspecs);
5065 catfp = BytesIO(catstr);
5066 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5067 return listcatfiles;
5069 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5071 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5072 formatspecs = FormatSpecsListToDict(formatspecs);
5073 catfp = BytesIO();
5074 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5075 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5076 return listcatfiles;
5078 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5079 formatspecs = FormatSpecsListToDict(formatspecs);
5080 catfp = BytesIO();
5081 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5082 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5083 return listcatfiles;
5085 if(not rarfile_support):
5086 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5087 return False;
5089 if(rarfile_support):
5090 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5091 formatspecs = FormatSpecsListToDict(formatspecs);
5092 catfp = BytesIO();
5093 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5094 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5095 return listcatfiles;
5097 if(not py7zr_support):
5098 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5099 return False;
5101 if(py7zr_support):
5102 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5103 formatspecs = FormatSpecsListToDict(formatspecs);
5104 catfp = BytesIO();
5105 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5106 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5107 return listcatfiles;
5109 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5110 formatspecs = FormatSpecsListToDict(formatspecs);
5111 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5112 if(checkcompressfile=="tarfile"):
5113 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5114 elif(checkcompressfile=="zipfile"):
5115 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5116 elif(checkcompressfile=="catfile"):
5117 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5118 elif(rarfile_support and checkcompressfile=="rarfile"):
5119 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5120 elif(py7zr_support and checkcompressfile=="7zipfile"):
5121 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5122 else:
5123 return False;
5124 return False;
5126 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5127 formatspecs = FormatSpecsListToDict(formatspecs);
5128 catver = formatspecs['format_ver'];
5129 fileheaderver = str(int(catver.replace(".", "")));
5130 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5131 advancedlist = formatspecs['use_advanced_list'];
5132 altinode = formatspecs['use_alt_inode'];
5133 infilelist = [];
5134 if(infiles=="-"):
5135 for line in sys.stdin:
5136 infilelist.append(line.strip());
5137 infilelist = list(filter(None, infilelist));
5138 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5139 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5140 return False;
5141 with UncompressFile(infiles, formatspecs, "r") as finfile:
5142 for line in finfile:
5143 infilelist.append(line.strip());
5144 infilelist = list(filter(None, infilelist));
5145 else:
5146 if(isinstance(infiles, (list, tuple, ))):
5147 infilelist = list(filter(None, infiles));
5148 elif(isinstance(infiles, (str, ))):
5149 infilelist = list(filter(None, [infiles]));
5150 if(advancedlist):
5151 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5152 else:
5153 GetDirList = ListDir(infilelist, followlink, False);
5154 if(not GetDirList):
5155 return False;
5156 curinode = 0;
5157 curfid = 0;
5158 inodelist = [];
5159 inodetofile = {};
5160 filetoinode = {};
5161 inodetocatinode = {};
5162 fileidnum = 0;
5163 fnumfiles = int(len(GetDirList));
5164 catver = formatspecs['format_ver'];
5165 fileheaderver = str(int(catver.replace(".", "")));
5166 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5167 fnumfileshex = format(int(fnumfiles), 'x').lower();
5168 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5169 catversion = re.findall(r"([\d]+)", fileheader);
5170 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5171 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5172 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5173 fheadtell = len(fileheader);
5174 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5175 for curfname in GetDirList:
5176 catfhstart = fheadtell;
5177 if(re.findall(r"^[.|/]", curfname)):
5178 fname = curfname;
5179 else:
5180 fname = "./"+curfname;
5181 if(verbose):
5182 VerbosePrintOut(fname);
5183 if(not followlink or followlink is None):
5184 fstatinfo = os.lstat(fname);
5185 else:
5186 fstatinfo = os.stat(fname);
5187 fpremode = fstatinfo.st_mode;
5188 finode = fstatinfo.st_ino;
5189 flinkcount = fstatinfo.st_nlink;
5190 ftype = 0;
5191 if(stat.S_ISREG(fpremode)):
5192 ftype = 0;
5193 elif(stat.S_ISLNK(fpremode)):
5194 ftype = 2;
5195 elif(stat.S_ISCHR(fpremode)):
5196 ftype = 3;
5197 elif(stat.S_ISBLK(fpremode)):
5198 ftype = 4;
5199 elif(stat.S_ISDIR(fpremode)):
5200 ftype = 5;
5201 elif(stat.S_ISFIFO(fpremode)):
5202 ftype = 6;
5203 elif(stat.S_ISSOCK(fpremode)):
5204 ftype = 8;
5205 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5206 ftype = 9;
5207 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5208 ftype = 10;
5209 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5210 ftype = 11;
5211 else:
5212 ftype = 0;
5213 flinkname = "";
5214 fbasedir = os.path.dirname(fname);
5215 fcurfid = curfid;
5216 if(not followlink and finode!=0):
5217 if(ftype!=1):
5218 if(finode in inodelist):
5219 ftype = 1;
5220 flinkname = inodetofile[finode];
5221 if(altinode):
5222 fcurinode = finode;
5223 else:
5224 fcurinode = inodetocatinode[finode];
5225 if(finode not in inodelist):
5226 inodelist.append(finode);
5227 inodetofile.update({finode: fname});
5228 inodetocatinode.update({finode: curinode});
5229 if(altinode):
5230 fcurinode = finode;
5231 else:
5232 fcurinode = curinode;
5233 curinode = curinode + 1;
5234 else:
5235 fcurinode = curinode;
5236 curinode = curinode + 1;
5237 curfid = curfid + 1;
5238 if(ftype==2):
5239 flinkname = os.readlink(fname);
5240 fdev = fstatinfo.st_dev;
5241 getfdev = GetDevMajorMinor(fdev);
5242 fdev_minor = getfdev[0];
5243 fdev_major = getfdev[1];
5244 frdev = fstatinfo.st_dev;
5245 if(hasattr(fstatinfo, "st_rdev")):
5246 frdev = fstatinfo.st_rdev;
5247 else:
5248 frdev = fstatinfo.st_dev;
5249 getfrdev = GetDevMajorMinor(frdev);
5250 frdev_minor = getfrdev[0];
5251 frdev_major = getfrdev[1];
5252 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5253 fsize = "0";
5254 if(ftype==0 or ftype==7):
5255 fsize = fstatinfo.st_size;
5256 fatime = fstatinfo.st_atime;
5257 fmtime = fstatinfo.st_mtime;
5258 fctime = fstatinfo.st_ctime;
5259 if(hasattr(fstatinfo, "st_birthtime")):
5260 fbtime = fstatinfo.st_birthtime;
5261 else:
5262 fbtime = fstatinfo.st_ctime;
5263 fmode = fstatinfo.st_mode;
5264 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5265 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5266 fuid = fstatinfo.st_uid;
5267 fgid = fstatinfo.st_gid;
5268 funame = "";
5269 try:
5270 import pwd;
5271 try:
5272 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5273 funame = userinfo.pw_name;
5274 except KeyError:
5275 funame = "";
5276 except ImportError:
5277 funame = "";
5278 fgname = "";
5279 try:
5280 import grp;
5281 try:
5282 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5283 fgname = groupinfo.gr_name;
5284 except KeyError:
5285 fgname = "";
5286 except ImportError:
5287 fgname = "";
5288 fdev_minor = fdev_minor;
5289 fdev_major = fdev_major;
5290 frdev_minor = frdev_minor;
5291 frdev_major = frdev_major;
5292 flinkcount = flinkcount;
5293 if(hasattr(fstatinfo, "st_file_attributes")):
5294 fwinattributes = fstatinfo.st_file_attributes;
5295 else:
5296 fwinattributes = 0;
5297 fcompression = "";
5298 fcsize = 0;
5299 fcontents = BytesIO();
5300 if(ftype==0 or ftype==7):
5301 with open(fname, "rb") as fpc:
5302 shutil.copyfileobj(fpc, fcontents);
5303 if(followlink and (ftype==1 or ftype==2)):
5304 flstatinfo = os.stat(flinkname);
5305 with open(flinkname, "rb") as fpc:
5306 shutil.copyfileobj(fpc, fcontents);
5307 fcontents.seek(0, 0);
5308 ftypehex = format(ftype, 'x').lower();
5309 extrafields = len(extradata);
5310 extrafieldslist = extradata;
5311 catfextrafields = extrafields;
5312 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5313 if(len(extradata)>0):
5314 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5315 extrasizelen = len(extrasizestr);
5316 extrasizelenhex = format(extrasizelen, 'x').lower();
5317 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5318 catoutlen = len(catoutlist) + len(extradata) + 3;
5319 catoutlenhex = format(catoutlen, 'x').lower();
5320 catoutlist.insert(0, catoutlenhex);
5321 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5322 catheaderdata = catoutlist;
5323 if(len(extradata)>0):
5324 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5325 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5326 catfnumfields = catoutlen;
5327 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5328 fcontents.seek(0, 0);
5329 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5330 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5331 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5332 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5333 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5334 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5335 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5336 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5337 catfcontentstart = fheadtell;
5338 fheadtell += len(catfileoutstr) + 1;
5339 catfcontentend = fheadtell - 1;
5340 catfhend = catfcontentend;
5341 fcontents.seek(0, 0);
5342 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5343 pyhascontents = False;
5344 if(int(fsize)>0 and not listonly):
5345 pyhascontents = True;
5346 if(int(fsize)>0 and listonly):
5347 fcontents = BytesIO();
5348 pyhascontents = False;
5349 fcontents.seek(0, 0);
5350 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5351 fileidnum = fileidnum + 1;
5352 return catlist;
5354 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5355 formatspecs = FormatSpecsListToDict(formatspecs);
5356 curinode = 0;
5357 curfid = 0;
5358 inodelist = [];
5359 inodetofile = {};
5360 filetoinode = {};
5361 inodetocatinode = {};
5362 fileidnum = 0;
5363 if(infile=="-"):
5364 infile = BytesIO();
5365 if(hasattr(sys.stdin, "buffer")):
5366 shutil.copyfileobj(sys.stdin.buffer, infile);
5367 else:
5368 shutil.copyfileobj(sys.stdin, infile);
5369 infile.seek(0, 0);
5370 if(not infile):
5371 return False;
5372 infile.seek(0, 0);
5373 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5374 infile = download_file_from_internet_file(infile);
5375 infile.seek(0, 0);
5376 if(not infile):
5377 return False;
5378 infile.seek(0, 0);
5379 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5380 return False;
5381 elif(os.path.exists(infile) and os.path.isfile(infile)):
5382 try:
5383 if(not tarfile.TarFileCheck(infile)):
5384 return False;
5385 except AttributeError:
5386 if(not TarFileCheck(infile)):
5387 return False;
5388 else:
5389 return False;
5390 try:
5391 if(hasattr(infile, "read") or hasattr(infile, "write")):
5392 tarfp = tarfile.open(fileobj=infile, mode="r");
5393 else:
5394 tarfp = tarfile.open(infile, "r");
5395 except FileNotFoundError:
5396 return False;
5397 fnumfiles = int(len(tarfp.getmembers()));
5398 catver = formatspecs['format_ver'];
5399 fileheaderver = str(int(catver.replace(".", "")));
5400 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5401 fnumfileshex = format(int(fnumfiles), 'x').lower();
5402 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5403 catversion = re.findall(r"([\d]+)", fileheader);
5404 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5405 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5406 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5407 fheadtell = len(fileheader);
5408 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5409 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5410 catfhstart = fheadtell;
5411 if(re.findall(r"^[.|/]", member.name)):
5412 fname = member.name;
5413 else:
5414 fname = "./"+member.name;
5415 if(verbose):
5416 VerbosePrintOut(fname);
5417 fpremode = member.mode;
5418 ffullmode = member.mode;
5419 flinkcount = 0;
5420 ftype = 0;
5421 if(member.isreg()):
5422 ffullmode = member.mode + stat.S_IFREG;
5423 ftype = 0;
5424 elif(member.isdev()):
5425 ffullmode = member.mode;
5426 ftype = 7;
5427 elif(member.islnk()):
5428 ffullmode = member.mode + stat.S_IFREG;
5429 ftype = 1;
5430 elif(member.issym()):
5431 ffullmode = member.mode + stat.S_IFLNK;
5432 ftype = 2;
5433 elif(member.ischr()):
5434 ffullmode = member.mode + stat.S_IFCHR;
5435 ftype = 3;
5436 elif(member.isblk()):
5437 ffullmode = member.mode + stat.S_IFBLK;
5438 ftype = 4;
5439 elif(member.isdir()):
5440 ffullmode = member.mode + stat.S_IFDIR;
5441 ftype = 5;
5442 elif(member.isfifo()):
5443 ffullmode = member.mode + stat.S_IFIFO;
5444 ftype = 6;
5445 elif(member.issparse()):
5446 ffullmode = member.mode;
5447 ftype = 12;
5448 else:
5449 ffullmode = member.mode;
5450 ftype = 0;
5451 flinkname = "";
5452 fbasedir = os.path.dirname(fname);
5453 fcurfid = curfid;
5454 fcurinode = curfid;
5455 finode = fcurinode;
5456 curfid = curfid + 1;
5457 if(ftype==2):
5458 flinkname = member.linkname;
5459 fdev_minor = member.devminor;
5460 fdev_major = member.devmajor;
5461 frdev_minor = member.devminor;
5462 frdev_major = member.devmajor;
5463 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5464 fsize = "0";
5465 elif(ftype==0 or ftype==7):
5466 fsize = member.size;
5467 else:
5468 fsize = member.size;
5469 fatime = member.mtime;
5470 fmtime = member.mtime;
5471 fctime = member.mtime;
5472 fbtime = member.mtime;
5473 fmode = ffullmode;
5474 fchmode = stat.S_IMODE(ffullmode);
5475 ftypemod = stat.S_IFMT(ffullmode);
5476 fuid = member.uid;
5477 fgid = member.gid;
5478 funame = member.uname;
5479 fgname = member.gname;
5480 flinkcount = flinkcount;
5481 fwinattributes = int(0);
5482 fcompression = "";
5483 fcsize = 0;
5484 fcontents = BytesIO();
5485 if(ftype==0 or ftype==7):
5486 with tarfp.extractfile(member) as fpc:
5487 shutil.copyfileobj(fpc, fcontents);
5488 fcontents.seek(0, 0);
5489 ftypehex = format(ftype, 'x').lower();
5490 extrafields = len(extradata);
5491 extrafieldslist = extradata;
5492 catfextrafields = extrafields;
5493 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5494 if(len(extradata)>0):
5495 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5496 extrasizelen = len(extrasizestr);
5497 extrasizelenhex = format(extrasizelen, 'x').lower();
5498 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5499 catoutlen = len(catoutlist) + len(extradata) + 3;
5500 catoutlenhex = format(catoutlen, 'x').lower();
5501 catoutlist.insert(0, catoutlenhex);
5502 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5503 catheaderdata = catoutlist;
5504 if(len(extradata)>0):
5505 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5506 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5507 catfnumfields = catoutlen;
5508 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5509 fcontents.seek(0, 0);
5510 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5511 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5512 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5513 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5514 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5515 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5516 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5517 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5518 catfcontentstart = fheadtell;
5519 fheadtell += len(catfileoutstr) + 1;
5520 catfcontentend = fheadtell - 1;
5521 catfhend = catfcontentend;
5522 fcontents.seek(0, 0);
5523 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5524 pyhascontents = False;
5525 if(int(fsize)>0 and not listonly):
5526 pyhascontents = True;
5527 if(int(fsize)>0 and listonly):
5528 fcontents = BytesIO();
5529 pyhascontents = False;
5530 fcontents.seek(0, 0);
5531 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5532 fileidnum = fileidnum + 1;
5533 return catlist;
5535 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5536 formatspecs = FormatSpecsListToDict(formatspecs);
5537 curinode = 0;
5538 curfid = 0;
5539 inodelist = [];
5540 inodetofile = {};
5541 filetoinode = {};
5542 inodetocatinode = {};
5543 fileidnum = 0;
5544 if(infile=="-"):
5545 infile = BytesIO();
5546 if(hasattr(sys.stdin, "buffer")):
5547 shutil.copyfileobj(sys.stdin.buffer, infile);
5548 else:
5549 shutil.copyfileobj(sys.stdin, infile);
5550 infile.seek(0, 0);
5551 if(not infile):
5552 return False;
5553 infile.seek(0, 0);
5554 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5555 infile = download_file_from_internet_file(infile);
5556 infile.seek(0, 0);
5557 if(not infile):
5558 return False;
5559 infile.seek(0, 0);
5560 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5561 return False;
5562 else:
5563 return False;
5564 if(not zipfile.is_zipfile(infile)):
5565 return False;
5566 try:
5567 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5568 except FileNotFoundError:
5569 return False;
5570 ziptest = zipfp.testzip();
5571 if(ziptest):
5572 VerbosePrintOut("Bad file found!");
5573 fnumfiles = int(len(zipfp.infolist()));
5574 catver = formatspecs['format_ver'];
5575 fileheaderver = str(int(catver.replace(".", "")));
5576 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5577 catversion = re.findall(r"([\d]+)", fileheader);
5578 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5579 fnumfileshex = format(int(fnumfiles), 'x').lower();
5580 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5581 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5582 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5583 fheadtell = len(fileheader);
5584 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5585 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5586 catfhstart = fheadtell;
5587 if(re.findall(r"^[.|/]", member.filename)):
5588 fname = member.filename;
5589 else:
5590 fname = "./"+member.filename;
5591 zipinfo = zipfp.getinfo(member.filename);
5592 if(verbose):
5593 VerbosePrintOut(fname);
5594 if(not member.is_dir()):
5595 fpremode = stat.S_IFREG + 438;
5596 elif(member.is_dir()):
5597 fpremode = stat.S_IFDIR + 511;
5598 flinkcount = 0;
5599 ftype = 0;
5600 if(not member.is_dir()):
5601 ftype = 0;
5602 elif(member.is_dir()):
5603 ftype = 5;
5604 flinkname = "";
5605 fbasedir = os.path.dirname(fname);
5606 fcurfid = curfid;
5607 fcurinode = curfid;
5608 finode = fcurinode;
5609 curfid = curfid + 1;
5610 fdev_minor = 0;
5611 fdev_major = 0;
5612 frdev_minor = 0;
5613 frdev_major = 0;
5614 if(ftype==5):
5615 fsize = "0";
5616 elif(ftype==0):
5617 fsize = member.file_size;
5618 else:
5619 fsize = member.file_size;
5620 fatime = time.mktime(member.date_time + (0, 0, -1));
5621 fmtime = time.mktime(member.date_time + (0, 0, -1));
5622 fctime = time.mktime(member.date_time + (0, 0, -1));
5623 fbtime = time.mktime(member.date_time + (0, 0, -1));
5624 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5625 fwinattributes = int(zipinfo.external_attr);
5626 if(not member.is_dir()):
5627 fmode = int(stat.S_IFREG + 438);
5628 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5629 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5630 elif(member.is_dir()):
5631 fmode = int(stat.S_IFDIR + 511);
5632 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5633 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5634 elif(zipinfo.create_system==3):
5635 fwinattributes = int(0);
5636 fmode = int(zipinfo.external_attr);
5637 else:
5638 fwinattributes = int(0);
5639 if(not member.is_dir()):
5640 fmode = int(stat.S_IFREG + 438);
5641 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5642 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5643 elif(member.is_dir()):
5644 fmode = int(stat.S_IFDIR + 511);
5645 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5646 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5647 fcompression = "";
5648 fcsize = 0;
5649 try:
5650 fuid = os.getuid();
5651 except AttributeError:
5652 fuid = 0;
5653 except KeyError:
5654 fuid = 0;
5655 try:
5656 fgid = os.getgid();
5657 except AttributeError:
5658 fgid = 0;
5659 except KeyError:
5660 fgid = 0;
5661 try:
5662 import pwd;
5663 try:
5664 userinfo = pwd.getpwuid(os.getuid());
5665 funame = userinfo.pw_name;
5666 except KeyError:
5667 funame = "";
5668 except AttributeError:
5669 funame = "";
5670 except ImportError:
5671 funame = "";
5672 fgname = "";
5673 try:
5674 import grp;
5675 try:
5676 groupinfo = grp.getgrgid(os.getgid());
5677 fgname = groupinfo.gr_name;
5678 except KeyError:
5679 fgname = "";
5680 except AttributeError:
5681 fgname = "";
5682 except ImportError:
5683 fgname = "";
5684 fcontents = BytesIO();
5685 if(ftype==0):
5686 fcontents.write(zipfp.read(member.filename));
5687 fcontents.seek(0, 0);
5688 ftypehex = format(ftype, 'x').lower();
5689 extrafields = len(extradata);
5690 extrafieldslist = extradata;
5691 catfextrafields = extrafields;
5692 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5693 if(len(extradata)>0):
5694 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5695 extrasizelen = len(extrasizestr);
5696 extrasizelenhex = format(extrasizelen, 'x').lower();
5697 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5698 catoutlen = len(catoutlist) + len(extradata) + 3;
5699 catoutlenhex = format(catoutlen, 'x').lower();
5700 catoutlist.insert(0, catoutlenhex);
5701 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5702 catheaderdata = catoutlist;
5703 if(len(extradata)>0):
5704 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5705 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5706 catfnumfields = catoutlen;
5707 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5708 fcontents.seek(0, 0);
5709 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5710 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5711 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5712 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5713 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5714 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5715 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5716 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5717 catfcontentstart = fheadtell;
5718 fheadtell += len(catfileoutstr) + 1;
5719 catfcontentend = fheadtell - 1;
5720 catfhend = catfcontentend;
5721 fcontents.seek(0, 0);
5722 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5723 pyhascontents = False;
5724 if(int(fsize)>0 and not listonly):
5725 pyhascontents = True;
5726 if(int(fsize)>0 and listonly):
5727 fcontents = BytesIO();
5728 pyhascontents = False;
5729 fcontents.seek(0, 0);
5730 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5731 fileidnum = fileidnum + 1;
5732 return catlist;
5734 if(not rarfile_support):
5735 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5736 return False;
5738 if(rarfile_support):
5739 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5740 formatspecs = FormatSpecsListToDict(formatspecs);
5741 curinode = 0;
5742 curfid = 0;
5743 inodelist = [];
5744 inodetofile = {};
5745 filetoinode = {};
5746 inodetocatinode = {};
5747 fileidnum = 0;
5748 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5749 return False;
5750 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5751 return False;
5752 rarfp = rarfile.RarFile(infile, "r");
5753 rartest = rarfp.testrar();
5754 if(rartest):
5755 VerbosePrintOut("Bad file found!");
5756 fnumfiles = int(len(rarfp.infolist()));
5757 catver = formatspecs['format_ver'];
5758 fileheaderver = str(int(catver.replace(".", "")));
5759 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5760 catversion = re.findall(r"([\d]+)", fileheader);
5761 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5762 fnumfileshex = format(int(fnumfiles), 'x').lower();
5763 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5764 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5765 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5766 fheadtell = len(fileheader);
5767 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5768 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5769 catfhstart = fheadtell;
5770 is_unix = False;
5771 is_windows = False;
5772 if(member.host_os==rarfile.RAR_OS_UNIX):
5773 is_windows = False;
5774 try:
5775 member.external_attr
5776 is_unix = True;
5777 except AttributeError:
5778 is_unix = False;
5779 elif(member.host_os==rarfile.RAR_OS_WIN32):
5780 is_unix = False;
5781 try:
5782 member.external_attr
5783 is_windows = True;
5784 except AttributeError:
5785 is_windows = False;
5786 else:
5787 is_unix = False;
5788 is_windows = False;
5789 if(re.findall(r"^[.|/]", member.filename)):
5790 fname = member.filename;
5791 else:
5792 fname = "./"+member.filename;
5793 rarinfo = rarfp.getinfo(member.filename);
5794 if(verbose):
5795 VerbosePrintOut(fname);
5796 if(is_unix and member.external_attr !=0):
5797 fpremode = int(member.external_attr);
5798 elif(member.is_file()):
5799 fpremode = stat.S_IFREG + 438;
5800 elif(member.is_symlink()):
5801 fpremode = stat.S_IFLNK + 438;
5802 elif(member.is_dir()):
5803 fpremode = stat.S_IFDIR + 511;
5804 if(is_windows and member.external_attr !=0):
5805 fwinattributes = int(member.external_attr);
5806 else:
5807 fwinattributes = int(0);
5808 fcompression = "";
5809 fcsize = 0;
5810 flinkcount = 0;
5811 ftype = 0;
5812 if(member.is_file()):
5813 ftype = 0;
5814 elif(member.is_symlink()):
5815 ftype = 2;
5816 elif(member.is_dir()):
5817 ftype = 5;
5818 flinkname = "";
5819 if(ftype==2):
5820 flinkname = rarfp.read(member.filename).decode("UTF-8");
5821 fbasedir = os.path.dirname(fname);
5822 fcurfid = curfid;
5823 fcurinode = curfid;
5824 finode = fcurinode;
5825 curfid = curfid + 1;
5826 fdev_minor = 0;
5827 fdev_major = 0;
5828 frdev_minor = 0;
5829 frdev_major = 0;
5830 if(ftype==5):
5831 fsize = "0";
5832 if(ftype==0):
5833 fsize = member.file_size;
5834 try:
5835 if(member.atime):
5836 fatime = int(member.atime.timestamp());
5837 else:
5838 fatime = int(member.mtime.timestamp());
5839 except AttributeError:
5840 fatime = int(member.mtime.timestamp());
5841 fmtime = int(member.mtime.timestamp());
5842 try:
5843 if(member.ctime):
5844 fctime = int(member.ctime.timestamp());
5845 else:
5846 fctime = int(member.mtime.timestamp());
5847 except AttributeError:
5848 fctime = int(member.mtime.timestamp());
5849 fbtime = int(member.mtime.timestamp());
5850 if(is_unix and member.external_attr !=0):
5851 fmode = int(member.external_attr);
5852 fchmode = int(stat.S_IMODE(member.external_attr));
5853 ftypemod = int(stat.S_IFMT(member.external_attr));
5854 elif(member.is_file()):
5855 fmode = int(stat.S_IFREG + 438)
5856 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5857 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5858 elif(member.is_symlink()):
5859 fmode = int(stat.S_IFLNK + 438)
5860 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5861 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5862 elif(member.is_dir()):
5863 fmode = int(stat.S_IFDIR + 511)
5864 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5865 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5866 try:
5867 fuid = os.getuid();
5868 except AttributeError:
5869 fuid = 0;
5870 except KeyError:
5871 fuid = 0;
5872 try:
5873 fgid = os.getgid();
5874 except AttributeError:
5875 fgid = 0;
5876 except KeyError:
5877 fgid = 0;
5878 try:
5879 import pwd;
5880 try:
5881 userinfo = pwd.getpwuid(os.getuid());
5882 funame = userinfo.pw_name;
5883 except KeyError:
5884 funame = "";
5885 except AttributeError:
5886 funame = "";
5887 except ImportError:
5888 funame = "";
5889 fgname = "";
5890 try:
5891 import grp;
5892 try:
5893 groupinfo = grp.getgrgid(os.getgid());
5894 fgname = groupinfo.gr_name;
5895 except KeyError:
5896 fgname = "";
5897 except AttributeError:
5898 fgname = "";
5899 except ImportError:
5900 fgname = "";
5901 fcontents = BytesIO();
5902 if(ftype==0):
5903 fcontents.write(rarfp.read(member.filename));
5904 fcontents.seek(0, 0);
5905 ftypehex = format(ftype, 'x').lower();
5906 extrafields = len(extradata);
5907 extrafieldslist = extradata;
5908 catfextrafields = extrafields;
5909 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5910 if(len(extradata)>0):
5911 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5912 extrasizelen = len(extrasizestr);
5913 extrasizelenhex = format(extrasizelen, 'x').lower();
5914 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5915 catoutlen = len(catoutlist) + len(extradata) + 3;
5916 catoutlenhex = format(catoutlen, 'x').lower();
5917 catoutlist.insert(0, catoutlenhex);
5918 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5919 if(len(extradata)>0):
5920 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5921 ccatfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5922 catfnumfields = 24 + catfextrafields;
5923 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5924 fcontents.seek(0, 0);
5925 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5926 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5927 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5928 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5929 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5930 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5931 catheaderdata = catoutlist;
5932 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5933 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5934 catfcontentstart = fheadtell;
5935 fheadtell += len(catfileoutstr) + 1;
5936 catfcontentend = fheadtell - 1;
5937 catfhend = catfcontentend;
5938 fcontents.seek(0, 0);
5939 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5940 pyhascontents = False;
5941 if(int(fsize)>0 and not listonly):
5942 pyhascontents = True;
5943 if(int(fsize)>0 and listonly):
5944 fcontents = BytesIO();
5945 pyhascontents = False;
5946 fcontents.seek(0, 0);
5947 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5948 fileidnum = fileidnum + 1;
5949 return catlist;
5951 if(not py7zr_support):
5952 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5953 return False;
5955 if(py7zr_support):
5956 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5957 formatspecs = FormatSpecsListToDict(formatspecs);
5958 curinode = 0;
5959 curfid = 0;
5960 inodelist = [];
5961 inodetofile = {};
5962 filetoinode = {};
5963 inodetocatinode = {};
5964 fileidnum = 0;
5965 szpfp = py7zr.SevenZipFile(infile, mode="r");
5966 file_content = szpfp.readall();
5967 #sztest = szpfp.testzip();
5968 sztestalt = szpfp.test();
5969 if(sztestalt):
5970 VerbosePrintOut("Bad file found!");
5971 numfiles = int(len(szpfp.list()));
5972 catver = formatspecs['format_ver'];
5973 fileheaderver = str(int(catver.replace(".", "")));
5974 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5975 catversion = re.findall(r"([\d]+)", fileheader);
5976 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5977 fnumfileshex = format(int(fnumfiles), 'x').lower();
5978 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5979 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5980 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5981 fheadtell = len(fileheader);
5982 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5983 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5984 catfhstart = fheadtell;
5985 if(re.findall(r"^[.|/]", member.filename)):
5986 fname = member.filename;
5987 else:
5988 fname = "./"+member.filename;
5989 if(not member.is_directory):
5990 fpremode = int(stat.S_IFREG + 438);
5991 elif(member.is_directory):
5992 fpremode = int(stat.S_IFDIR + 511);
5993 fwinattributes = int(0);
5994 fcompression = "";
5995 fcsize = 0;
5996 flinkcount = 0;
5997 ftype = 0;
5998 if(member.is_directory):
5999 ftype = 5;
6000 else:
6001 ftype = 0;
6002 flinkname = "";
6003 fbasedir = os.path.dirname(fname);
6004 fcurfid = curfid;
6005 fcurinode = curfid;
6006 finode = fcurinode;
6007 curfid = curfid + 1;
6008 fdev_minor = 0;
6009 fdev_major = 0;
6010 frdev_minor = 0;
6011 frdev_major = 0;
6012 if(ftype==5):
6013 fsize = "0";
6014 fatime = int(member.creationtime.timestamp());
6015 fmtime = int(member.creationtime.timestamp());
6016 fctime = int(member.creationtime.timestamp());
6017 fbtime = int(member.creationtime.timestamp());
6018 if(member.is_directory):
6019 fmode = int(stat.S_IFDIR + 511)
6020 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6021 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6022 else:
6023 fmode = int(stat.S_IFLNK + 438)
6024 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6025 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6026 try:
6027 fuid = os.getuid();
6028 except AttributeError:
6029 fuid = 0;
6030 except KeyError:
6031 fuid = 0;
6032 try:
6033 fgid = os.getgid();
6034 except AttributeError:
6035 fgid = 0;
6036 except KeyError:
6037 fgid = 0;
6038 try:
6039 import pwd;
6040 try:
6041 userinfo = pwd.getpwuid(os.getuid());
6042 funame = userinfo.pw_name;
6043 except KeyError:
6044 funame = "";
6045 except AttributeError:
6046 funame = "";
6047 except ImportError:
6048 funame = "";
6049 fgname = "";
6050 try:
6051 import grp;
6052 try:
6053 groupinfo = grp.getgrgid(os.getgid());
6054 fgname = groupinfo.gr_name;
6055 except KeyError:
6056 fgname = "";
6057 except AttributeError:
6058 fgname = "";
6059 except ImportError:
6060 fgname = "";
6061 fcontents = BytesIO();
6062 if(ftype==0):
6063 fcontents.write(file_content[member.filename].read());
6064 fsize = format(fcontents.tell(), 'x').lower();
6065 fileop.close();
6066 fcontents.seek(0, 0);
6067 ftypehex = format(ftype, 'x').lower();
6068 extrafields = len(extradata);
6069 extrafieldslist = extradata;
6070 catfextrafields = extrafields;
6071 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6072 if(len(extradata)>0):
6073 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6074 extrasizelen = len(extrasizestr);
6075 extrasizelenhex = format(extrasizelen, 'x').lower();
6076 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6077 catoutlen = len(catoutlist) + len(extradata) + 3;
6078 catoutlenhex = format(catoutlen, 'x').lower();
6079 catoutlist.insert(0, catoutlenhex);
6080 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6081 catheaderdata = catoutlist;
6082 if(len(extradata)>0):
6083 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6084 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
6085 catfnumfields = 24 + catfextrafields;
6086 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6087 fcontents.seek(0, 0);
6088 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6089 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6090 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6091 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6092 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6093 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6094 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6095 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6096 catfcontentstart = fheadtell;
6097 fheadtell += len(catfileoutstr) + 1;
6098 catfcontentend = fheadtell - 1;
6099 catfhend = catfcontentend;
6100 fcontents.seek(0, 0);
6101 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6102 pyhascontents = False;
6103 if(int(fsize)>0 and not listonly):
6104 pyhascontents = True;
6105 if(int(fsize)>0 and listonly):
6106 fcontents = BytesIO();
6107 pyhascontents = False;
6108 fcontents.seek(0, 0);
6109 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6110 fileidnum = fileidnum + 1;
6111 return catlist;
6113 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6114 formatspecs = FormatSpecsListToDict(formatspecs);
6115 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6116 if(checkcompressfile=="tarfile"):
6117 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6118 elif(checkcompressfile=="zipfile"):
6119 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6120 elif(checkcompressfile=="catfile"):
6121 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6122 elif(rarfile_support and checkcompressfile=="rarfile"):
6123 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6124 elif(py7zr_support and checkcompressfile=="7zipfile"):
6125 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6126 else:
6127 return False;
6128 return False;
6130 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6131 formatspecs = FormatSpecsListToDict(formatspecs);
6132 outarray = BytesIO();
6133 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6134 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6135 return listcatfiles;
6137 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6138 formatspecs = FormatSpecsListToDict(formatspecs);
6139 if(isinstance(inarray, dict)):
6140 listcatfiles = inarray;
6141 else:
6142 return False;
6143 if(not listcatfiles):
6144 return False;
6145 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6146 if(returnfp):
6147 catarray.update({'catfp': listcatfiles['catfp']});
6148 lenlist = len(listcatfiles['ffilelist']);
6149 lcfi = 0;
6150 lcfx = int(listcatfiles['fnumfiles']);
6151 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6152 lcfx = int(lenlist);
6153 else:
6154 lcfx = int(listcatfiles['fnumfiles']);
6155 while(lcfi < lcfx):
6156 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6157 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6158 catarray['filetoid'].update(filetoidarray);
6159 catarray['idtofile'].update(idtofilearray);
6160 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6161 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6162 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6163 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6164 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6165 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6166 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6167 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6168 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6169 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6170 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6171 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6172 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6173 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6174 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6175 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6176 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6177 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6178 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6179 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6180 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6181 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6182 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6183 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6184 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6185 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6186 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6187 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6188 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6189 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6190 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6191 lcfi = lcfi + 1;
6192 return catarray;
6194 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6196 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6197 formatspecs = FormatSpecsListToDict(formatspecs);
6198 if(isinstance(infile, dict)):
6199 listcatfiles = infile;
6200 else:
6201 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6202 infile = RemoveWindowsPath(infile);
6203 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6204 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6205 outfile = RemoveWindowsPath(outfile);
6206 checksumtype = checksumtype.lower();
6207 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6208 checksumtype="crc32";
6209 if(checksumtype=="none"):
6210 checksumtype = "";
6211 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6212 compression = "auto";
6213 if(compression not in compressionlist and compression is None):
6214 compression = "auto";
6215 if(verbose):
6216 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6217 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6218 if(os.path.exists(outfile)):
6219 try:
6220 os.unlink(outfile);
6221 except OSError as e:
6222 pass;
6223 if(not listcatfiles):
6224 return False;
6225 if(outfile=="-"):
6226 verbose = False;
6227 catfp = BytesIO();
6228 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6229 catfp = outfile;
6230 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6231 catfp = BytesIO();
6232 else:
6233 fbasename = os.path.splitext(outfile)[0];
6234 fextname = os.path.splitext(outfile)[1];
6235 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6236 catver = formatspecs['format_ver'];
6237 fileheaderver = str(int(catver.replace(".", "")));
6238 lenlist = len(listcatfiles['ffilelist']);
6239 fnumfiles = int(listcatfiles['fnumfiles']);
6240 if(lenlist>fnumfiles or lenlist<fnumfiles):
6241 fnumfiles = lenlist;
6242 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6243 lenlist = len(listcatfiles['ffilelist']);
6244 fnumfiles = int(listcatfiles['fnumfiles']);
6245 lcfi = 0;
6246 lcfx = int(listcatfiles['fnumfiles']);
6247 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6248 lcfx = int(lenlist);
6249 else:
6250 lcfx = int(listcatfiles['fnumfiles']);
6251 curinode = 0;
6252 curfid = 0;
6253 inodelist = [];
6254 inodetofile = {};
6255 filetoinode = {};
6256 reallcfi = 0;
6257 while(lcfi < lcfx):
6258 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6259 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6260 else:
6261 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6262 if(verbose):
6263 VerbosePrintOut(fname);
6264 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6265 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6266 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6267 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6268 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6269 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6270 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6271 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6272 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6273 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6274 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6275 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6276 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6277 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6278 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6279 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6280 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6281 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6282 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6283 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6284 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6285 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6286 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6287 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6288 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6289 if(not followlink and len(extradata)<0):
6290 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6291 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6292 fcompression = "";
6293 fcsize = format(int(0), 'x').lower();
6294 if(not compresswholefile):
6295 fcontents.seek(0, 2);
6296 ucfsize = fcontents.tell();
6297 fcontents.seek(0, 0);
6298 if(compression=="auto"):
6299 ilsize = len(compressionlistalt);
6300 ilmin = 0;
6301 ilcsize = [];
6302 while(ilmin < ilsize):
6303 cfcontents = BytesIO();
6304 shutil.copyfileobj(fcontents, cfcontents);
6305 fcontents.seek(0, 0);
6306 cfcontents.seek(0, 0);
6307 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6308 if(cfcontents):
6309 cfcontents.seek(0, 2);
6310 ilcsize.append(cfcontents.tell());
6311 cfcontents.close();
6312 else:
6313 try:
6314 ilcsize.append(sys.maxint);
6315 except AttributeError:
6316 ilcsize.append(sys.maxsize);
6317 ilmin = ilmin + 1;
6318 ilcmin = ilcsize.index(min(ilcsize));
6319 compression = compressionlistalt[ilcmin];
6320 fcontents.seek(0, 0);
6321 cfcontents = BytesIO();
6322 shutil.copyfileobj(fcontents, cfcontents);
6323 cfcontents.seek(0, 0);
6324 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6325 cfcontents.seek(0, 2);
6326 cfsize = cfcontents.tell();
6327 if(ucfsize > cfsize):
6328 fcsize = format(int(cfsize), 'x').lower();
6329 fcompression = compression;
6330 fcontents.close();
6331 fcontents = cfcontents;
6332 if(followlink):
6333 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6334 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6335 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6336 flinkinfo = listcatfiles['ffilelist'][flinkid];
6337 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6338 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6339 flinkname = flinkinfo['flinkname'];
6340 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6341 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6342 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6343 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6344 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6345 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6346 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6347 funame = flinkinfo['funame'];
6348 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6349 fgname = flinkinfo['fgname'];
6350 finode = format(int(flinkinfo['finode']), 'x').lower();
6351 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6352 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6353 fcompression = flinkinfo['fcompression'];
6354 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6355 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6356 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6357 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6358 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6359 fseeknextfile = flinkinfo['fseeknextfile'];
6360 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6361 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6362 if(len(extradata)<0):
6363 extradata = flinkinfo['fextralist'];
6364 fcontents = flinkinfo['fcontents'];
6365 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6366 else:
6367 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6368 fcurfid = format(curfid, 'x').lower();
6369 if(not followlink and finode!=0):
6370 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6371 fcurinode = format(int(curinode), 'x').lower();
6372 inodetofile.update({curinode: fname});
6373 filetoinode.update({fname: curinode});
6374 curinode = curinode + 1;
6375 else:
6376 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6377 else:
6378 fcurinode = format(int(curinode), 'x').lower();
6379 curinode = curinode + 1;
6380 curfid = curfid + 1;
6381 if(fcompression=="none"):
6382 fcompression = "";
6383 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6384 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6385 fcontents.close();
6386 lcfi = lcfi + 1;
6387 reallcfi = reallcfi + 1;
6388 if(lcfx>0):
6389 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6390 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6391 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6392 try:
6393 catfp.flush();
6394 os.fsync(catfp.fileno());
6395 except io.UnsupportedOperation:
6396 pass;
6397 except AttributeError:
6398 pass;
6399 except OSError as e:
6400 pass;
6401 if(outfile=="-"):
6402 catfp.seek(0, 0);
6403 if(hasattr(sys.stdout, "buffer")):
6404 shutil.copyfileobj(catfp, sys.stdout.buffer);
6405 else:
6406 shutil.copyfileobj(catfp, sys.stdout);
6407 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6408 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6409 catfp.seek(0, 0);
6410 upload_file_to_internet_file(catfp, outfile);
6411 if(returnfp):
6412 catfp.seek(0, 0);
6413 return catfp;
6414 else:
6415 catfp.close();
6416 return True;
6418 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6420 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6421 formatspecs = FormatSpecsListToDict(formatspecs);
6422 catfp = BytesIO(catstr);
6423 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6424 return listcatfiles;
6426 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6428 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6429 formatspecs = FormatSpecsListToDict(formatspecs);
6430 outarray = BytesIO();
6431 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6432 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6433 return listcatfiles;
6435 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6437 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6438 formatspecs = FormatSpecsListToDict(formatspecs);
6439 if(outdir is not None):
6440 outdir = RemoveWindowsPath(outdir);
6441 if(verbose):
6442 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6443 if(isinstance(infile, dict)):
6444 listcatfiles = infile;
6445 else:
6446 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6447 infile = RemoveWindowsPath(infile);
6448 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6449 if(not listcatfiles):
6450 return False;
6451 lenlist = len(listcatfiles['ffilelist']);
6452 fnumfiles = int(listcatfiles['fnumfiles']);
6453 lcfi = 0;
6454 lcfx = int(listcatfiles['fnumfiles']);
6455 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6456 lcfx = int(lenlist);
6457 else:
6458 lcfx = int(listcatfiles['fnumfiles']);
6459 while(lcfi<lcfx):
6460 funame = "";
6461 try:
6462 import pwd;
6463 try:
6464 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6465 funame = userinfo.pw_name;
6466 except KeyError:
6467 funame = "";
6468 except ImportError:
6469 funame = "";
6470 fgname = "";
6471 try:
6472 import grp;
6473 try:
6474 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6475 fgname = groupinfo.gr_name;
6476 except KeyError:
6477 fgname = "";
6478 except ImportError:
6479 fgname = "";
6480 if(verbose):
6481 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6482 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6483 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6484 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6485 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6486 try:
6487 fpc.flush();
6488 os.fsync(fpc.fileno());
6489 except io.UnsupportedOperation:
6490 pass;
6491 except AttributeError:
6492 pass;
6493 except OSError as e:
6494 pass;
6495 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6496 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6497 if(preservepermissions):
6498 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6499 if(preservetime):
6500 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6501 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6502 if(followlink):
6503 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6504 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6505 flinkinfo = listcatfiles['ffilelist'][flinkid];
6506 funame = "";
6507 try:
6508 import pwd;
6509 try:
6510 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6511 funame = userinfo.pw_name;
6512 except KeyError:
6513 funame = "";
6514 except ImportError:
6515 funame = "";
6516 fgname = "";
6517 try:
6518 import grp;
6519 try:
6520 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6521 fgname = groupinfo.gr_name;
6522 except KeyError:
6523 fgname = "";
6524 except ImportError:
6525 fgname = "";
6526 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6527 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6528 flinkinfo['fcontents'].seek(0, 0);
6529 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6530 try:
6531 fpc.flush();
6532 os.fsync(fpc.fileno());
6533 except io.UnsupportedOperation:
6534 pass;
6535 except AttributeError:
6536 pass;
6537 except OSError as e:
6538 pass;
6539 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6540 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6541 if(preservepermissions):
6542 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6543 if(preservetime):
6544 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6545 if(flinkinfo['ftype']==1):
6546 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6547 if(flinkinfo['ftype']==2):
6548 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6549 if(flinkinfo['ftype']==5):
6550 if(preservepermissions):
6551 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6552 else:
6553 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6554 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6555 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6556 if(preservepermissions):
6557 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6558 if(preservetime):
6559 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6560 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6561 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6562 else:
6563 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6564 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6565 if(followlink):
6566 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6567 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6568 flinkinfo = listcatfiles['ffilelist'][flinkid];
6569 funame = "";
6570 try:
6571 import pwd;
6572 try:
6573 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6574 funame = userinfo.pw_name;
6575 except KeyError:
6576 funame = "";
6577 except ImportError:
6578 funame = "";
6579 fgname = "";
6580 try:
6581 import grp;
6582 try:
6583 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6584 fgname = groupinfo.gr_name;
6585 except KeyError:
6586 fgname = "";
6587 except ImportError:
6588 fgname = "";
6589 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6590 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6591 flinkinfo['fcontents'].seek(0, 0);
6592 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6593 try:
6594 fpc.flush();
6595 os.fsync(fpc.fileno());
6596 except io.UnsupportedOperation:
6597 pass;
6598 except AttributeError:
6599 pass;
6600 except OSError as e:
6601 pass;
6602 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6603 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6604 if(preservepermissions):
6605 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6606 if(preservetime):
6607 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6608 if(flinkinfo['ftype']==1):
6609 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6610 if(flinkinfo['ftype']==2):
6611 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6612 if(flinkinfo['ftype']==5):
6613 if(preservepermissions):
6614 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6615 else:
6616 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6617 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6618 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6619 if(preservepermissions):
6620 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6621 if(preservetime):
6622 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6623 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6624 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6625 else:
6626 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6627 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6628 if(preservepermissions):
6629 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6630 else:
6631 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6632 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6633 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6634 if(preservepermissions):
6635 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6636 if(preservetime):
6637 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6638 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6639 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6640 lcfi = lcfi + 1;
6641 if(returnfp):
6642 return listcatfiles['ffilelist']['catfp'];
6643 else:
6644 return True;
6646 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6648 if(hasattr(shutil, "register_unpack_format")):
6649 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6650 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6651 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6653 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6654 formatspecs = FormatSpecsListToDict(formatspecs);
6655 catfp = BytesIO(catstr);
6656 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6657 return listcatfiles;
6659 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6661 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6662 formatspecs = FormatSpecsListToDict(formatspecs);
6663 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6664 if(isinstance(infile, dict)):
6665 listcatfiles = infile;
6666 else:
6667 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6668 infile = RemoveWindowsPath(infile);
6669 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6670 if(not listcatfiles):
6671 return False;
6672 lenlist = len(listcatfiles['ffilelist']);
6673 fnumfiles = int(listcatfiles['fnumfiles']);
6674 lcfi = 0;
6675 lcfx = int(listcatfiles['fnumfiles']);
6676 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6677 lcfx = int(lenlist);
6678 else:
6679 lcfx = int(listcatfiles['fnumfiles']);
6680 returnval = {};
6681 while(lcfi<lcfx):
6682 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6683 if(not verbose):
6684 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6685 if(verbose):
6686 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6687 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6688 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6689 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6690 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6691 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6692 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6693 if(len(fuprint)<=0):
6694 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6695 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6696 if(len(fgprint)<=0):
6697 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6698 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6699 lcfi = lcfi + 1;
6700 if(returnfp):
6701 return listcatfiles['catfp'];
6702 else:
6703 return True;
6705 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6707 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6708 formatspecs = FormatSpecsListToDict(formatspecs);
6709 catfp = BytesIO(catstr);
6710 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6711 return listcatfiles;
6713 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6715 def TarFileListFiles(infile, verbose=False, returnfp=False):
6716 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6717 if(infile=="-"):
6718 infile = BytesIO();
6719 if(hasattr(sys.stdin, "buffer")):
6720 shutil.copyfileobj(sys.stdin.buffer, infile);
6721 else:
6722 shutil.copyfileobj(sys.stdin, infile);
6723 infile.seek(0, 0);
6724 if(not infile):
6725 return False;
6726 infile.seek(0, 0);
6727 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6728 infile = download_file_from_internet_file(infile);
6729 infile.seek(0, 0);
6730 if(not infile):
6731 return False;
6732 infile.seek(0, 0);
6733 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6734 return False;
6735 elif(os.path.exists(infile) and os.path.isfile(infile)):
6736 try:
6737 if(not tarfile.TarFileCheck(infile)):
6738 return False;
6739 except AttributeError:
6740 if(not TarFileCheck(infile)):
6741 return False;
6742 else:
6743 return False;
6744 try:
6745 if(hasattr(infile, "read") or hasattr(infile, "write")):
6746 tarfp = tarfile.open(fileobj=infile, mode="r");
6747 else:
6748 tarfp = tarfile.open(infile, "r");
6749 except FileNotFoundError:
6750 return False;
6751 lcfi = 0
6752 returnval = {};
6753 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6754 returnval.update({lcfi: member.name});
6755 fpremode = member.mode;
6756 ffullmode = member.mode;
6757 flinkcount = 0;
6758 ftype = 0;
6759 if(member.isreg()):
6760 ffullmode = member.mode + stat.S_IFREG;
6761 ftype = 0;
6762 elif(member.isdev()):
6763 ffullmode = member.mode;
6764 ftype = 7;
6765 elif(member.islnk()):
6766 ffullmode = member.mode + stat.S_IFREG;
6767 ftype = 1;
6768 elif(member.issym()):
6769 ffullmode = member.mode + stat.S_IFLNK;
6770 ftype = 2;
6771 elif(member.ischr()):
6772 ffullmode = member.mode + stat.S_IFCHR;
6773 ftype = 3;
6774 elif(member.isblk()):
6775 ffullmode = member.mode + stat.S_IFBLK;
6776 ftype = 4;
6777 elif(member.isdir()):
6778 ffullmode = member.mode + stat.S_IFDIR;
6779 ftype = 5;
6780 elif(member.isfifo()):
6781 ffullmode = member.mode + stat.S_IFIFO;
6782 ftype = 6;
6783 elif(member.issparse()):
6784 ffullmode = member.mode;
6785 ftype = 12;
6786 if(not verbose):
6787 VerbosePrintOut(member.name);
6788 elif(verbose):
6789 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6790 printfname = member.name;
6791 if(member.islnk()):
6792 printfname = member.name + " link to " + member.linkname;
6793 elif(member.issym()):
6794 printfname = member.name + " -> " + member.linkname;
6795 fuprint = member.uname;
6796 if(len(fuprint)<=0):
6797 fuprint = member.uid;
6798 fgprint = member.gname;
6799 if(len(fgprint)<=0):
6800 fgprint = member.gid;
6801 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6802 lcfi = lcfi + 1;
6803 if(returnfp):
6804 return listcatfiles['catfp'];
6805 else:
6806 return True;
6808 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6809 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6810 if(infile=="-"):
6811 infile = BytesIO();
6812 if(hasattr(sys.stdin, "buffer")):
6813 shutil.copyfileobj(sys.stdin.buffer, infile);
6814 else:
6815 shutil.copyfileobj(sys.stdin, infile);
6816 infile.seek(0, 0);
6817 if(not infile):
6818 return False;
6819 infile.seek(0, 0);
6820 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6821 infile = download_file_from_internet_file(infile);
6822 infile.seek(0, 0);
6823 if(not infile):
6824 return False;
6825 infile.seek(0, 0);
6826 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6827 return False;
6828 else:
6829 return False;
6830 if(not zipfile.is_zipfile(infile)):
6831 return False;
6832 try:
6833 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6834 except FileNotFoundError:
6835 return False;
6836 lcfi = 0;
6837 returnval = {};
6838 ziptest = zipfp.testzip();
6839 if(ziptest):
6840 VerbosePrintOut("Bad file found!");
6841 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6842 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6843 fwinattributes = int(zipinfo.external_attr);
6844 if(not member.is_dir()):
6845 fmode = int(stat.S_IFREG + 438);
6846 fchmode = int(stat.S_IMODE(fmode));
6847 ftypemod = int(stat.S_IFMT(fmode));
6848 elif(member.is_dir()):
6849 fmode = int(stat.S_IFDIR + 511);
6850 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6851 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6852 elif(zipinfo.create_system==3):
6853 fwinattributes =int(0);
6854 fmode = int(zipinfo.external_attr);
6855 fchmode = int(stat.S_IMODE(fmode));
6856 ftypemod = int(stat.S_IFMT(fmode));
6857 else:
6858 fwinattributes = int(0);
6859 if(not member.is_dir()):
6860 fmode = int(stat.S_IFREG + 438);
6861 fchmode = int(stat.S_IMODE(fmode));
6862 ftypemod = int(stat.S_IFMT(fmode));
6863 elif(member.is_dir()):
6864 fmode = int(stat.S_IFDIR + 511);
6865 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6866 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6867 returnval.update({lcfi: member.filename});
6868 if(not verbose):
6869 VerbosePrintOut(member.filename);
6870 if(verbose):
6871 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6872 permissionstr = "";
6873 for fmodval in str(oct(fmode))[-3:]:
6874 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6875 if(not member.is_dir()):
6876 ftype = 0;
6877 permissionstr = "-" + permissionstr;
6878 elif(member.is_dir()):
6879 ftype = 5;
6880 permissionstr = "d" + permissionstr;
6881 printfname = member.filename;
6882 try:
6883 fuid = int(os.getuid());
6884 except AttributeError:
6885 fuid = int(0);
6886 except KeyError:
6887 fuid = int(0);
6888 try:
6889 fgid = int(os.getgid());
6890 except AttributeError:
6891 fgid = int(0);
6892 except KeyError:
6893 fgid = int(0);
6894 try:
6895 import pwd;
6896 try:
6897 userinfo = pwd.getpwuid(os.getuid());
6898 funame = userinfo.pw_name;
6899 except KeyError:
6900 funame = "";
6901 except AttributeError:
6902 funame = "";
6903 except ImportError:
6904 funame = "";
6905 fgname = "";
6906 try:
6907 import grp;
6908 try:
6909 groupinfo = grp.getgrgid(os.getgid());
6910 fgname = groupinfo.gr_name;
6911 except KeyError:
6912 fgname = "";
6913 except AttributeError:
6914 fgname = "";
6915 except ImportError:
6916 fgname = "";
6917 fuprint = funame;
6918 if(len(fuprint)<=0):
6919 fuprint = str(fuid);
6920 fgprint = fgname;
6921 if(len(fgprint)<=0):
6922 fgprint = str(fgid);
6923 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6924 lcfi = lcfi + 1;
6925 if(returnfp):
6926 return listcatfiles['catfp'];
6927 else:
6928 return True;
6930 if(not rarfile_support):
6931 def RarFileListFiles(infile, verbose=False, returnfp=False):
6932 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6933 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6934 return False;
6936 if(rarfile_support):
6937 def RarFileListFiles(infile, verbose=False, returnfp=False):
6938 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6939 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6940 return False;
6941 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6942 return False;
6943 lcfi = 0;
6944 returnval = {};
6945 rarfp = rarfile.RarFile(infile, "r");
6946 rartest = rarfp.testrar();
6947 if(rartest):
6948 VerbosePrintOut("Bad file found!");
6949 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6950 is_unix = False;
6951 is_windows = False;
6952 if(member.host_os==rarfile.RAR_OS_UNIX):
6953 is_windows = False;
6954 try:
6955 member.external_attr
6956 is_unix = True;
6957 except AttributeError:
6958 is_unix = False;
6959 elif(member.host_os==rarfile.RAR_OS_WIN32):
6960 is_unix = False;
6961 try:
6962 member.external_attr
6963 is_windows = True;
6964 except AttributeError:
6965 is_windows = False;
6966 else:
6967 is_unix = False;
6968 is_windows = False;
6969 if(is_unix and member.external_attr !=0):
6970 fpremode = int(member.external_attr);
6971 elif(member.is_file()):
6972 fpremode = int(stat.S_IFREG + 438);
6973 elif(member.is_symlink()):
6974 fpremode = int(stat.S_IFLNK + 438);
6975 elif(member.is_dir()):
6976 fpremode = int(stat.S_IFDIR + 511);
6977 if(is_windows and member.external_attr !=0):
6978 fwinattributes = int(member.external_attr);
6979 else:
6980 fwinattributes = int(0);
6981 if(is_unix and member.external_attr !=0):
6982 fmode = int(member.external_attr);
6983 fchmode = int(stat.S_IMODE(member.external_attr));
6984 ftypemod = int(stat.S_IFMT(member.external_attr));
6985 elif(member.is_file()):
6986 fmode = int(stat.S_IFREG + 438);
6987 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6988 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6989 elif(member.is_symlink()):
6990 fmode = int(stat.S_IFLNK + 438);
6991 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6992 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6993 elif(member.is_dir()):
6994 fmode = int(stat.S_IFDIR + 511);
6995 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6996 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6997 returnval.update({lcfi: member.filename});
6998 if(not verbose):
6999 VerbosePrintOut(member.filename);
7000 if(verbose):
7001 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7002 permissionstr = "";
7003 for fmodval in str(oct(fmode))[-3:]:
7004 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7005 if(member.is_file()):
7006 ftype = 0;
7007 permissionstr = "-" + permissionstr;
7008 printfname = member.filename;
7009 elif(member.is_symlink()):
7010 ftype = 2;
7011 permissionstr = "l" + permissionstr;
7012 printfname = member.name + " -> " + member.read().decode("UTF-8");
7013 elif(member.is_dir()):
7014 ftype = 5;
7015 permissionstr = "d" + permissionstr;
7016 printfname = member.filename;
7017 try:
7018 fuid = int(os.getuid());
7019 except AttributeError:
7020 fuid = int(0);
7021 except KeyError:
7022 fuid = int(0);
7023 try:
7024 fgid = int(os.getgid());
7025 except AttributeError:
7026 fgid = int(0);
7027 except KeyError:
7028 fgid = int(0);
7029 try:
7030 import pwd;
7031 try:
7032 userinfo = pwd.getpwuid(os.getuid());
7033 funame = userinfo.pw_name;
7034 except KeyError:
7035 funame = "";
7036 except AttributeError:
7037 funame = "";
7038 except ImportError:
7039 funame = "";
7040 fgname = "";
7041 try:
7042 import grp;
7043 try:
7044 groupinfo = grp.getgrgid(os.getgid());
7045 fgname = groupinfo.gr_name;
7046 except KeyError:
7047 fgname = "";
7048 except AttributeError:
7049 fgname = "";
7050 except ImportError:
7051 fgname = "";
7052 fuprint = funame;
7053 if(len(fuprint)<=0):
7054 fuprint = str(fuid);
7055 fgprint = fgname;
7056 if(len(fgprint)<=0):
7057 fgprint = str(fgid);
7058 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7059 lcfi = lcfi + 1;
7060 if(returnfp):
7061 return listcatfiles['catfp'];
7062 else:
7063 return True;
7065 if(not py7zr_support):
7066 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7067 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7068 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7069 return False;
7071 if(py7zr_support):
7072 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7073 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7074 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7075 return False;
7076 lcfi = 0;
7077 returnval = {};
7078 szpfp = py7zr.SevenZipFile(infile, mode="r");
7079 file_content = szpfp.readall();
7080 #sztest = szpfp.testzip();
7081 sztestalt = szpfp.test();
7082 if(sztestalt):
7083 VerbosePrintOut("Bad file found!");
7084 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7085 if(re.findall(r"^[.|/]", member.filename)):
7086 fname = member.filename;
7087 else:
7088 fname = "./"+member.filename;
7089 if(not member.is_directory):
7090 fpremode = int(stat.S_IFREG + 438);
7091 elif(member.is_directory):
7092 fpremode = int(stat.S_IFDIR + 511);
7093 fwinattributes = int(0);
7094 if(member.is_directory):
7095 fmode = int(stat.S_IFDIR + 511);
7096 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7097 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7098 else:
7099 fmode = int(stat.S_IFLNK + 438);
7100 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7101 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7102 returnval.update({lcfi: member.filename});
7103 if(not verbose):
7104 VerbosePrintOut(member.filename);
7105 if(verbose):
7106 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7107 permissionstr = "";
7108 for fmodval in str(oct(fmode))[-3:]:
7109 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7110 fsize = int("0");
7111 if(not member.is_directory):
7112 ftype = 0;
7113 permissionstr = "-" + permissionstr;
7114 printfname = member.filename;
7115 elif(member.is_directory):
7116 ftype = 5;
7117 permissionstr = "d" + permissionstr;
7118 printfname = member.filename;
7119 if(ftype==0):
7120 fsize = len(file_content[member.filename].read());
7121 file_content[member.filename].close();
7122 try:
7123 fuid = int(os.getuid());
7124 except AttributeError:
7125 fuid = int(0);
7126 except KeyError:
7127 fuid = int(0);
7128 try:
7129 fgid = int(os.getgid());
7130 except AttributeError:
7131 fgid = int(0);
7132 except KeyError:
7133 fgid = int(0);
7134 try:
7135 import pwd;
7136 try:
7137 userinfo = pwd.getpwuid(os.getuid());
7138 funame = userinfo.pw_name;
7139 except KeyError:
7140 funame = "";
7141 except AttributeError:
7142 funame = "";
7143 except ImportError:
7144 funame = "";
7145 fgname = "";
7146 try:
7147 import grp;
7148 try:
7149 groupinfo = grp.getgrgid(os.getgid());
7150 fgname = groupinfo.gr_name;
7151 except KeyError:
7152 fgname = "";
7153 except AttributeError:
7154 fgname = "";
7155 except ImportError:
7156 fgname = "";
7157 fuprint = funame;
7158 if(len(fuprint)<=0):
7159 fuprint = str(fuid);
7160 fgprint = fgname;
7161 if(len(fgprint)<=0):
7162 fgprint = str(fgid);
7163 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7164 lcfi = lcfi + 1;
7165 if(returnfp):
7166 return listcatfiles['catfp'];
7167 else:
7168 return True;
7170 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7171 formatspecs = FormatSpecsListToDict(formatspecs);
7172 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7173 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7174 if(checkcompressfile=="tarfile"):
7175 return TarFileListFiles(infile, verbose, returnfp);
7176 elif(checkcompressfile=="zipfile"):
7177 return ZipFileListFiles(infile, verbose, returnfp);
7178 elif(checkcompressfile=="catfile"):
7179 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7180 elif(rarfile_support and checkcompressfile=="rarfile"):
7181 return RarFileListFiles(infile, verbose, returnfp);
7182 elif(py7zr_support and checkcompressfile=="7zipfile"):
7183 return SevenZipFileListFiles(infile, verbose, returnfp);
7184 else:
7185 return False;
7186 return False;
7188 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7189 formatspecs = FormatSpecsListToDict(formatspecs);
7190 outarray = BytesIO();
7191 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7192 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7193 return listcatfiles;
7195 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7196 formatspecs = FormatSpecsListToDict(formatspecs);
7197 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7198 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7199 return listcatfiles;
7201 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7202 formatspecs = FormatSpecsListToDict(formatspecs);
7203 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7204 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7205 return listcatfiles;
7207 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7209 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7210 formatspecs = FormatSpecsListToDict(formatspecs);
7211 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7212 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7213 return listcatfiles;
7215 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7217 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7218 formatspecs = FormatSpecsListToDict(formatspecs);
7219 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7220 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7221 return listcatfiles;
7223 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7225 if(not rarfile_support):
7226 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7227 return False;
7229 if(rarfile_support):
7230 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7231 formatspecs = FormatSpecsListToDict(formatspecs);
7232 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7233 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7234 return listcatfiles;
7236 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7238 if(not py7zr_support):
7239 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7240 return False;
7242 if(py7zr_support):
7243 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7244 formatspecs = FormatSpecsListToDict(formatspecs);
7245 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7246 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7247 return listcatfiles;
7249 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7251 def download_file_from_ftp_file(url):
7252 urlparts = urlparse(url);
7253 file_name = os.path.basename(urlparts.path);
7254 file_dir = os.path.dirname(urlparts.path);
7255 if(urlparts.username is not None):
7256 ftp_username = urlparts.username;
7257 else:
7258 ftp_username = "anonymous";
7259 if(urlparts.password is not None):
7260 ftp_password = urlparts.password;
7261 elif(urlparts.password is None and urlparts.username=="anonymous"):
7262 ftp_password = "anonymous";
7263 else:
7264 ftp_password = "";
7265 if(urlparts.scheme=="ftp"):
7266 ftp = FTP();
7267 elif(urlparts.scheme=="ftps" and ftpssl):
7268 ftp = FTP_TLS();
7269 else:
7270 return False;
7271 if(urlparts.scheme=="sftp"):
7272 if(__use_pysftp__):
7273 return download_file_from_pysftp_file(url);
7274 else:
7275 return download_file_from_sftp_file(url);
7276 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7277 return download_file_from_http_file(url);
7278 ftp_port = urlparts.port;
7279 if(urlparts.port is None):
7280 ftp_port = 21;
7281 try:
7282 ftp.connect(urlparts.hostname, ftp_port);
7283 except socket.gaierror:
7284 log.info("Error With URL "+url);
7285 return False;
7286 except socket.timeout:
7287 log.info("Error With URL "+url);
7288 return False;
7289 ftp.login(urlparts.username, urlparts.password);
7290 if(urlparts.scheme=="ftps"):
7291 ftp.prot_p();
7292 ftpfile = BytesIO();
7293 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7294 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7295 ftp.close();
7296 ftpfile.seek(0, 0);
7297 return ftpfile;
7299 def download_file_from_ftp_string(url):
7300 ftpfile = download_file_from_ftp_file(url);
7301 return ftpfile.read();
7303 def upload_file_to_ftp_file(ftpfile, url):
7304 urlparts = urlparse(url);
7305 file_name = os.path.basename(urlparts.path);
7306 file_dir = os.path.dirname(urlparts.path);
7307 if(urlparts.username is not None):
7308 ftp_username = urlparts.username;
7309 else:
7310 ftp_username = "anonymous";
7311 if(urlparts.password is not None):
7312 ftp_password = urlparts.password;
7313 elif(urlparts.password is None and urlparts.username=="anonymous"):
7314 ftp_password = "anonymous";
7315 else:
7316 ftp_password = "";
7317 if(urlparts.scheme=="ftp"):
7318 ftp = FTP();
7319 elif(urlparts.scheme=="ftps" and ftpssl):
7320 ftp = FTP_TLS();
7321 else:
7322 return False;
7323 if(urlparts.scheme=="sftp"):
7324 if(__use_pysftp__):
7325 return upload_file_to_pysftp_file(url);
7326 else:
7327 return upload_file_to_sftp_file(url);
7328 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7329 return False;
7330 ftp_port = urlparts.port;
7331 if(urlparts.port is None):
7332 ftp_port = 21;
7333 try:
7334 ftp.connect(urlparts.hostname, ftp_port);
7335 except socket.gaierror:
7336 log.info("Error With URL "+url);
7337 return False;
7338 except socket.timeout:
7339 log.info("Error With URL "+url);
7340 return False;
7341 ftp.login(urlparts.username, urlparts.password);
7342 if(urlparts.scheme=="ftps"):
7343 ftp.prot_p();
7344 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7345 ftp.close();
7346 ftpfile.seek(0, 0);
7347 return ftpfile;
7349 def upload_file_to_ftp_string(ftpstring, url):
7350 ftpfileo = BytesIO(ftpstring);
7351 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7352 ftpfileo.close();
7353 return ftpfile;
7355 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7356 # Parse the URL to extract username and password if present
7357 urlparts = urlparse(url);
7358 username = urlparts.username;
7359 password = urlparts.password;
7360 # Rebuild the URL without the username and password
7361 netloc = urlparts.hostname;
7362 if(urlparts.scheme=="sftp"):
7363 if(__use_pysftp__):
7364 return download_file_from_pysftp_file(url);
7365 else:
7366 return download_file_from_sftp_file(url);
7367 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7368 return download_file_from_ftp_file(url);
7369 if urlparts.port:
7370 netloc += ':' + str(urlparts.port);
7371 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7372 # Create a temporary file object
7373 httpfile = BytesIO();
7374 if haverequests:
7375 # Use the requests library if available
7376 if username and password:
7377 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7378 else:
7379 response = requests.get(rebuilt_url, headers=headers, stream=True);
7380 response.raw.decode_content = True
7381 shutil.copyfileobj(response.raw, httpfile);
7382 else:
7383 # Build a Request object for urllib
7384 request = Request(rebuilt_url, headers=headers);
7385 # Create an opener object for handling URLs
7386 if username and password:
7387 # Create a password manager
7388 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7389 # Add the username and password
7390 password_mgr.add_password(None, rebuilt_url, username, password);
7391 # Create an authentication handler using the password manager
7392 auth_handler = HTTPBasicAuthHandler(password_mgr);
7393 # Build the opener with the authentication handler
7394 opener = build_opener(auth_handler);
7395 else:
7396 opener = build_opener();
7397 with opener.open(request) as response:
7398 shutil.copyfileobj(response, httpfile);
7399 # Reset file pointer to the start
7400 httpfile.seek(0, 0);
7401 # Return the temporary file object
7402 return httpfile;
7404 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7405 httpfile = download_file_from_http_file(url, headers);
7406 return ftpfile.read();
7408 if(haveparamiko):
7409 def download_file_from_sftp_file(url):
7410 urlparts = urlparse(url);
7411 file_name = os.path.basename(urlparts.path);
7412 file_dir = os.path.dirname(urlparts.path);
7413 sftp_port = urlparts.port;
7414 if(urlparts.port is None):
7415 sftp_port = 22;
7416 else:
7417 sftp_port = urlparts.port;
7418 if(urlparts.username is not None):
7419 sftp_username = urlparts.username;
7420 else:
7421 sftp_username = "anonymous";
7422 if(urlparts.password is not None):
7423 sftp_password = urlparts.password;
7424 elif(urlparts.password is None and urlparts.username=="anonymous"):
7425 sftp_password = "anonymous";
7426 else:
7427 sftp_password = "";
7428 if(urlparts.scheme=="ftp"):
7429 return download_file_from_ftp_file(url);
7430 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7431 return download_file_from_http_file(url);
7432 if(urlparts.scheme!="sftp"):
7433 return False;
7434 ssh = paramiko.SSHClient();
7435 ssh.load_system_host_keys();
7436 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7437 try:
7438 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7439 except paramiko.ssh_exception.SSHException:
7440 return False;
7441 except socket.gaierror:
7442 log.info("Error With URL "+url);
7443 return False;
7444 except socket.timeout:
7445 log.info("Error With URL "+url);
7446 return False;
7447 sftp = ssh.open_sftp();
7448 sftpfile = BytesIO();
7449 sftp.getfo(urlparts.path, sftpfile);
7450 sftp.close();
7451 ssh.close();
7452 sftpfile.seek(0, 0);
7453 return sftpfile;
7454 else:
7455 def download_file_from_sftp_file(url):
7456 return False;
7458 if(haveparamiko):
7459 def download_file_from_sftp_string(url):
7460 sftpfile = download_file_from_sftp_file(url);
7461 return sftpfile.read();
7462 else:
7463 def download_file_from_ftp_string(url):
7464 return False;
7466 if(haveparamiko):
7467 def upload_file_to_sftp_file(sftpfile, url):
7468 urlparts = urlparse(url);
7469 file_name = os.path.basename(urlparts.path);
7470 file_dir = os.path.dirname(urlparts.path);
7471 sftp_port = urlparts.port;
7472 if(urlparts.port is None):
7473 sftp_port = 22;
7474 else:
7475 sftp_port = urlparts.port;
7476 if(urlparts.username is not None):
7477 sftp_username = urlparts.username;
7478 else:
7479 sftp_username = "anonymous";
7480 if(urlparts.password is not None):
7481 sftp_password = urlparts.password;
7482 elif(urlparts.password is None and urlparts.username=="anonymous"):
7483 sftp_password = "anonymous";
7484 else:
7485 sftp_password = "";
7486 if(urlparts.scheme=="ftp"):
7487 return upload_file_to_ftp_file(url);
7488 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7489 return False;
7490 if(urlparts.scheme!="sftp"):
7491 return False;
7492 ssh = paramiko.SSHClient();
7493 ssh.load_system_host_keys();
7494 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7495 try:
7496 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7497 except paramiko.ssh_exception.SSHException:
7498 return False;
7499 except socket.gaierror:
7500 log.info("Error With URL "+url);
7501 return False;
7502 except socket.timeout:
7503 log.info("Error With URL "+url);
7504 return False;
7505 sftp = ssh.open_sftp();
7506 sftp.putfo(sftpfile, urlparts.path);
7507 sftp.close();
7508 ssh.close();
7509 sftpfile.seek(0, 0);
7510 return sftpfile;
7511 else:
7512 def upload_file_to_sftp_file(sftpfile, url):
7513 return False;
7515 if(haveparamiko):
7516 def upload_file_to_sftp_string(sftpstring, url):
7517 sftpfileo = BytesIO(sftpstring);
7518 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7519 sftpfileo.close();
7520 return sftpfile;
7521 else:
7522 def upload_file_to_sftp_string(url):
7523 return False;
7525 if(havepysftp):
7526 def download_file_from_pysftp_file(url):
7527 urlparts = urlparse(url);
7528 file_name = os.path.basename(urlparts.path);
7529 file_dir = os.path.dirname(urlparts.path);
7530 sftp_port = urlparts.port;
7531 if(urlparts.port is None):
7532 sftp_port = 22;
7533 else:
7534 sftp_port = urlparts.port;
7535 if(urlparts.username is not None):
7536 sftp_username = urlparts.username;
7537 else:
7538 sftp_username = "anonymous";
7539 if(urlparts.password is not None):
7540 sftp_password = urlparts.password;
7541 elif(urlparts.password is None and urlparts.username=="anonymous"):
7542 sftp_password = "anonymous";
7543 else:
7544 sftp_password = "";
7545 if(urlparts.scheme=="ftp"):
7546 return download_file_from_ftp_file(url);
7547 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7548 return download_file_from_http_file(url);
7549 if(urlparts.scheme!="sftp"):
7550 return False;
7551 try:
7552 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7553 except paramiko.ssh_exception.SSHException:
7554 return False;
7555 except socket.gaierror:
7556 log.info("Error With URL "+url);
7557 return False;
7558 except socket.timeout:
7559 log.info("Error With URL "+url);
7560 return False;
7561 sftp = ssh.open_sftp();
7562 sftpfile = BytesIO();
7563 sftp.getfo(urlparts.path, sftpfile);
7564 sftp.close();
7565 ssh.close();
7566 sftpfile.seek(0, 0);
7567 return sftpfile;
7568 else:
7569 def download_file_from_pysftp_file(url):
7570 return False;
7572 if(havepysftp):
7573 def download_file_from_pysftp_string(url):
7574 sftpfile = download_file_from_pysftp_file(url);
7575 return sftpfile.read();
7576 else:
7577 def download_file_from_ftp_string(url):
7578 return False;
7580 if(havepysftp):
7581 def upload_file_to_pysftp_file(sftpfile, url):
7582 urlparts = urlparse(url);
7583 file_name = os.path.basename(urlparts.path);
7584 file_dir = os.path.dirname(urlparts.path);
7585 sftp_port = urlparts.port;
7586 if(urlparts.port is None):
7587 sftp_port = 22;
7588 else:
7589 sftp_port = urlparts.port;
7590 if(urlparts.username is not None):
7591 sftp_username = urlparts.username;
7592 else:
7593 sftp_username = "anonymous";
7594 if(urlparts.password is not None):
7595 sftp_password = urlparts.password;
7596 elif(urlparts.password is None and urlparts.username=="anonymous"):
7597 sftp_password = "anonymous";
7598 else:
7599 sftp_password = "";
7600 if(urlparts.scheme=="ftp"):
7601 return upload_file_to_ftp_file(url);
7602 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7603 return False;
7604 if(urlparts.scheme!="sftp"):
7605 return False;
7606 try:
7607 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7608 except paramiko.ssh_exception.SSHException:
7609 return False;
7610 except socket.gaierror:
7611 log.info("Error With URL "+url);
7612 return False;
7613 except socket.timeout:
7614 log.info("Error With URL "+url);
7615 return False;
7616 sftp = ssh.open_sftp();
7617 sftp.putfo(sftpfile, urlparts.path);
7618 sftp.close();
7619 ssh.close();
7620 sftpfile.seek(0, 0);
7621 return sftpfile;
7622 else:
7623 def upload_file_to_pysftp_file(sftpfile, url):
7624 return False;
7626 if(havepysftp):
7627 def upload_file_to_pysftp_string(sftpstring, url):
7628 sftpfileo = BytesIO(sftpstring);
7629 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7630 sftpfileo.close();
7631 return sftpfile;
7632 else:
7633 def upload_file_to_pysftp_string(url):
7634 return False;
7636 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7637 urlparts = urlparse(url);
7638 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7639 return download_file_from_http_file(url, headers);
7640 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7641 return download_file_from_ftp_file(url);
7642 elif(urlparts.scheme=="sftp"):
7643 if(__use_pysftp__ and havepysftp):
7644 return download_file_from_pysftp_file(url);
7645 else:
7646 return download_file_from_sftp_file(url);
7647 else:
7648 return False;
7649 return False;
7651 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7652 formatspecs = FormatSpecsListToDict(formatspecs);
7653 fp = download_file_from_internet_file(url);
7654 fp = UncompressArchiveFile(fp, formatspecs);
7655 fp.seek(0, 0);
7656 if(not fp):
7657 return False;
7658 return fp;
7660 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7661 urlparts = urlparse(url);
7662 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7663 return download_file_from_http_string(url, headers);
7664 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7665 return download_file_from_ftp_string(url);
7666 elif(urlparts.scheme=="sftp"):
7667 if(__use_pysftp__ and havepysftp):
7668 return download_file_from_pysftp_string(url);
7669 else:
7670 return download_file_from_sftp_string(url);
7671 else:
7672 return False;
7673 return False;
7675 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7676 formatspecs = FormatSpecsListToDict(formatspecs);
7677 fp = download_file_from_internet_string(url);
7678 fp = UncompressArchiveFile(fp, formatspecs);
7679 fp.seek(0, 0);
7680 if(not fp):
7681 return False;
7682 return fp;
7684 def upload_file_to_internet_file(ifp, url):
7685 urlparts = urlparse(url);
7686 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7687 return False;
7688 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7689 return upload_file_to_ftp_file(ifp, url);
7690 elif(urlparts.scheme=="sftp"):
7691 if(__use_pysftp__ and havepysftp):
7692 return upload_file_to_pysftp_file(ifp, url);
7693 else:
7694 return upload_file_to_sftp_file(ifp, url);
7695 else:
7696 return False;
7697 return False;
7699 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7700 formatspecs = FormatSpecsListToDict(formatspecs);
7701 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7702 if(not catfileout):
7703 return False;
7704 catfp.seek(0, 0);
7705 upload_file_to_internet_file(catfp, outfile);
7706 return True;
7708 def upload_file_to_internet_string(ifp, url):
7709 urlparts = urlparse(url);
7710 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7711 return False;
7712 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7713 return upload_file_to_ftp_string(ifp, url);
7714 elif(urlparts.scheme=="sftp"):
7715 if(__use_pysftp__ and havepysftp):
7716 return upload_file_to_pysftp_string(ifp, url);
7717 else:
7718 return upload_file_to_sftp_string(ifp, url);
7719 else:
7720 return False;
7721 return False;
7723 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7724 formatspecs = FormatSpecsListToDict(formatspecs);
7725 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7726 if(not catfileout):
7727 return False;
7728 catfp.seek(0, 0);
7729 upload_file_to_internet_file(catfp, outfile);
7730 return True;
7732 try:
7733 if(hasattr(shutil, "register_archive_format")):
7734 # Register the packing format
7735 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7736 except shutil.RegistryError:
7737 pass;
7739 try:
7740 if(hasattr(shutil, "register_unpack_format")):
7741 # Register the unpacking format
7742 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7743 except shutil.RegistryError:
7744 pass;