?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob08421f3d82901583005f4ce8695097e5ed1c2e11
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 6/26/2024 Ver. 0.13.12 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
38
39 try:
40 basestring;
41 except NameError:
42 basestring = str;
43
44 # URL Parsing
45 try:
46 from urllib.parse import urlparse, urlunparse;
47 except ImportError:
48 from urlparse import urlparse, urlunparse;
49
50 # Windows-specific setup
51 if os.name == 'nt':
52 if sys.version_info[0] == 2:
53 import codecs;
54 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
55 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
56 else:
57 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
58 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
59
60 hashlib_guaranteed = False;
61 # Environment setup
62 os.environ["PYTHONIOENCODING"] = "UTF-8";
63 os.environ["LC_CTYPE"] = "UTF-8";
64
65 # Reload sys to set default encoding to UTF-8 (Python 2 only)
66 if sys.version_info[0] == 2:
67 try:
68 reload(sys);
69 sys.setdefaultencoding('UTF-8');
70 except (NameError, AttributeError):
71 pass;
72
73 # CRC32 import
74 try:
75 from zlib import crc32;
76 except ImportError:
77 from binascii import crc32;
78
79 # Define FileNotFoundError for Python 2
80 try:
81 FileNotFoundError;
82 except NameError:
83 FileNotFoundError = IOError;
84
85 # RAR file support
86 rarfile_support = False;
87 try:
88 import rarfile;
89 rarfile_support = True;
90 except ImportError:
91 pass;
92
93 # 7z file support
94 py7zr_support = False;
95 try:
96 import py7zr;
97 py7zr_support = True;
98 except ImportError:
99 pass;
100
101 # TAR file checking
102 try:
103 from xtarfile import is_tarfile;
104 except ImportError:
105 try:
106 from safetar import is_tarfile;
107 except ImportError:
108 from tarfile import is_tarfile;
109
110 # TAR file module
111 try:
112 import xtarfile as tarfile;
113 except ImportError:
114 try:
115 import safetar as tarfile;
116 except ImportError:
117 import tarfile;
118
119 # Paramiko support
120 haveparamiko = False;
121 try:
122 import paramiko;
123 haveparamiko = True;
124 except ImportError:
125 pass;
126
127 # PySFTP support
128 havepysftp = False;
129 try:
130 import pysftp;
131 havepysftp = True;
132 except ImportError:
133 pass;
134
135 # Requests support
136 haverequests = False;
137 try:
138 import requests;
139 haverequests = True;
140 import urllib3;
141 logging.getLogger("urllib3").setLevel(logging.WARNING);
142 except ImportError:
143 pass;
144
145 # HTTPX support
146 havehttpx = False;
147 try:
148 import httpx;
149 havehttpx = True;
150 logging.getLogger("httpx").setLevel(logging.WARNING);
151 logging.getLogger("httpcore").setLevel(logging.WARNING);
152 except ImportError:
153 pass;
154
155 # HTTP and URL parsing
156 try:
157 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
158 from urllib.parse import urlparse;
159 except ImportError:
160 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
161 from urlparse import urlparse;
162
163 # StringIO and BytesIO
164 try:
165 from io import StringIO, BytesIO;
166 except ImportError:
167 try:
168 from cStringIO import StringIO;
169 from cStringIO import StringIO as BytesIO;
170 except ImportError:
171 from StringIO import StringIO;
172 from StringIO import StringIO as BytesIO;
173
174 __use_pysftp__ = False;
175 __use_alt_format__ = False;
176 if(not havepysftp):
177 __use_pysftp__ = False;
178 __use_http_lib__ = "httpx";
179 if(__use_http_lib__=="httpx" and haverequests and not havehttpx):
180 __use_http_lib__ = "requests";
181 if(__use_http_lib__=="requests" and havehttpx and not haverequests):
182 __use_http_lib__ = "httpx";
183 if((__use_http_lib__=="httpx" or __use_http_lib__=="requests") and not havehttpx and not haverequests):
184 __use_http_lib__ = "urllib";
185 if(not __use_alt_format__):
186 ''' Format Info by Kazuki Przyborowski '''
187 __file_format_name__ = "CatFile";
188 __program_name__ = "Py"+__file_format_name__;
189 __file_format_lower__ = __file_format_name__.lower();
190 __file_format_magic__ = __file_format_name__;
191 __file_format_len__ = len(__file_format_magic__);
192 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
193 __file_format_delimiter__ = "\x00";
194 __file_format_ver__ = "001";
195 __use_new_style__ = True;
196 __use_advanced_list__ = True;
197 __use_alt_inode__ = False;
198 __file_format_extension__ = ".cat";
199 else:
200 ''' Format Info Generated by ChatGPT '''
201 __file_format_name__ = "FastArchive";
202 __program_name__ = "Py" + __file_format_name__;
203 __file_format_lower__ = __file_format_name__.lower();
204 __file_format_magic__ = "FstArch";
205 __file_format_len__ = len(__file_format_magic__);
206 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
207 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
208 __file_format_ver__ = "001";
209 __use_new_style__ = True;
210 __use_advanced_list__ = False;
211 __use_alt_inode__ = False;
212 __file_format_extension__ = ".fast";
213 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
214 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
215 __project__ = __program_name__;
216 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
217 __version_info__ = (0, 13, 12, "RC 1", 1);
218 __version_date_info__ = (2024, 6, 26, "RC 1", 1);
219 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
220 __revision__ = __version_info__[3];
221 __revision_id__ = "$Id$";
222 if(__version_info__[4] is not None):
223 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
224 if(__version_info__[4] is None):
225 __version_date_plusrc__ = __version_date__;
226 if(__version_info__[3] is not None):
227 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
228 if(__version_info__[3] is None):
229 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
230
231 PyBitness = platform.architecture();
232 if(PyBitness=="32bit" or PyBitness=="32"):
233 PyBitness = "32";
234 elif(PyBitness=="64bit" or PyBitness=="64"):
235 PyBitness = "64";
236 else:
237 PyBitness = "32";
238
239 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
240 if(platform.python_implementation()!=""):
241 py_implementation = platform.python_implementation();
242 if(platform.python_implementation()==""):
243 py_implementation = "Python";
244 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
245 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
246 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
247 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
248 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
249 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
250 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
251
252 compressionsupport = [];
253 try:
254 import gzip;
255 compressionsupport.append("gz");
256 compressionsupport.append("gzip");
257 except ImportError:
258 pass;
259 try:
260 import bz2;
261 compressionsupport.append("bz2");
262 compressionsupport.append("bzip2");
263 except ImportError:
264 pass;
265 try:
266 import lz4, lz4.frame;
267 compressionsupport.append("lz4");
268 except ImportError:
269 pass;
270 try:
271 import lzo;
272 compressionsupport.append("lzo");
273 compressionsupport.append("lzop");
274 except ImportError:
275 pass;
276 try:
277 import zstandard;
278 compressionsupport.append("zstd");
279 compressionsupport.append("zstandard");
280 except ImportError:
281 pass;
282 try:
283 import lzma;
284 compressionsupport.append("lzma");
285 compressionsupport.append("xz");
286 except ImportError:
287 try:
288 from backports import lzma;
289 compressionsupport.append("lzma");
290 compressionsupport.append("xz");
291 except ImportError:
292 pass;
293 compressionsupport.append("zlib");
294
295 compressionlist = ['auto'];
296 compressionlistalt = [];
297 outextlist = [];
298 outextlistwd = [];
299 if('gzip' in compressionsupport):
300 compressionlist.append('gzip');
301 compressionlistalt.append('gzip');
302 outextlist.append('gz');
303 outextlistwd.append('.gz');
304 if('bzip2' in compressionsupport):
305 compressionlist.append('bzip2');
306 compressionlistalt.append('bzip2');
307 outextlist.append('bz2');
308 outextlistwd.append('.bz2');
309 if('zstd' in compressionsupport):
310 compressionlist.append('zstd');
311 compressionlistalt.append('zstd');
312 outextlist.append('zst');
313 outextlistwd.append('.zst');
314 if('lz4' in compressionsupport):
315 compressionlist.append('lz4');
316 compressionlistalt.append('lz4');
317 outextlist.append('lz4');
318 outextlistwd.append('.lz4');
319 if('lzo' in compressionsupport):
320 compressionlist.append('lzo');
321 compressionlistalt.append('lzo');
322 outextlist.append('lzo');
323 outextlistwd.append('.lzo');
324 if('lzop' in compressionsupport):
325 compressionlist.append('lzop');
326 compressionlistalt.append('lzop');
327 outextlist.append('lzop');
328 outextlistwd.append('.lzop');
329 if('lzma' in compressionsupport):
330 compressionlist.append('lzma');
331 compressionlistalt.append('lzma');
332 outextlist.append('lzma');
333 outextlistwd.append('.lzma');
334 if('xz' in compressionsupport):
335 compressionlist.append('xz');
336 compressionlistalt.append('xz');
337 outextlist.append('xz');
338 outextlistwd.append('.xz');
339 if('zlib' in compressionsupport):
340 compressionlist.append('zlib');
341 compressionlistalt.append('zlib');
342 outextlist.append('zz');
343 outextlistwd.append('.zz');
344 outextlist.append('zl');
345 outextlistwd.append('.zl');
346 outextlist.append('zlib');
347 outextlistwd.append('.zlib');
348
349 tarfile_mimetype = "application/tar";
350 tarfile_tar_mimetype = tarfile_mimetype;
351 zipfile_mimetype = "application/zip";
352 zipfile_zip_mimetype = zipfile_mimetype;
353 rarfile_mimetype = "application/rar";
354 rarfile_rar_mimetype = rarfile_mimetype;
355 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
356 archivefile_cat_mimetype = archivefile_mimetype;
357 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
358 archivefile_gz_mimetype = archivefile_gzip_mimetype;
359 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
360 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
361 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
362 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
363 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
364 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
365 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
366 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
367 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
368 archivefile_zlib_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zlib";
369 archivefile_zz_mimetype = archivefile_zlib_mimetype;
370 archivefile_zl_mimetype = archivefile_zlib_mimetype;
371 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz", __file_format_extension__+".zz", __file_format_extension__+".zl", __file_format_extension__+".zlib"];
372
373 if __name__ == "__main__":
374 import subprocess;
375 curscrpath = os.path.dirname(sys.argv[0]);
376 if(curscrpath==""):
377 curscrpath = ".";
378 if(os.sep=="\\"):
379 curscrpath = curscrpath.replace(os.sep, "/");
380 curscrpath = curscrpath + "/";
381 scrfile = curscrpath + "catfile.py";
382 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
383 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
384 scrcmd.wait();
385
386 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
387 if(not dbgenable):
388 return True;
389 log_functions = {
390 "print": print,
391 "log": logging.info,
392 "warning": logging.warning,
393 "error": logging.error,
394 "critical": logging.critical,
395 "exception": logging.exception,
396 "logalt": lambda x: logging.log(dgblevel, x),
397 "debug": logging.debug
398 };
399 log_function = log_functions.get(outtype);
400 if(log_function):
401 log_function(dbgtxt);
402 return True;
403 return False;
404
405 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
406 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
407 return dbgtxt;
408
409 def RemoveWindowsPath(dpath):
410 if(dpath is None):
411 dpath = "";
412 if(os.sep!="/"):
413 dpath = dpath.replace(os.path.sep, "/");
414 dpath = dpath.rstrip("/");
415 if(dpath=="." or dpath==".."):
416 dpath = dpath + "/";
417 return dpath;
418
419 def NormalizeRelativePath(inpath):
420 inpath = RemoveWindowsPath(inpath);
421 if(os.path.isabs(inpath)):
422 outpath = inpath;
423 else:
424 if(inpath.startswith("./") or inpath.startswith("../")):
425 outpath = inpath;
426 else:
427 outpath = "./" + inpath;
428 return outpath;
429
430 def PrependPath(base_dir, child_path):
431 # Check if base_dir is None or empty, if so, return child_path as is
432 if not base_dir:
433 return child_path;
434 # Ensure base_dir ends with exactly one slash
435 if not base_dir.endswith('/'):
436 base_dir += '/';
437 # Check if child_path starts with ./ or ../ (indicating a relative path)
438 if child_path.startswith('./') or child_path.startswith('../'):
439 # For relative paths, we don't alter the child_path
440 return base_dir + child_path;
441 else:
442 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
443 return base_dir + child_path.lstrip('/');
444
445 def ListDir(dirpath, followlink=False, duplicates=False):
446 if isinstance(dirpath, (list, tuple, )):
447 dirpath = list(filter(None, dirpath));
448 elif isinstance(dirpath, str):
449 dirpath = list(filter(None, [dirpath]));
450 retlist = [];
451 fs_encoding = sys.getfilesystemencoding();
452 for mydirfile in dirpath:
453 if not os.path.exists(mydirfile):
454 return False;
455 mydirfile = NormalizeRelativePath(mydirfile);
456 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
457 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
458 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
459 for root, dirs, filenames in os.walk(mydirfile):
460 dpath = root;
461 dpath = RemoveWindowsPath(dpath);
462 if fs_encoding != 'utf-8':
463 dpath = dpath.encode(fs_encoding).decode('utf-8');
464 if dpath not in retlist and not duplicates:
465 retlist.append(dpath);
466 if duplicates:
467 retlist.append(dpath);
468 for file in filenames:
469 fpath = os.path.join(root, file);
470 fpath = RemoveWindowsPath(fpath);
471 if fs_encoding != 'utf-8':
472 fpath = fpath.encode(fs_encoding).decode('utf-8');
473 if fpath not in retlist and not duplicates:
474 retlist.append(fpath);
475 if duplicates:
476 retlist.append(fpath);
477 else:
478 path = RemoveWindowsPath(mydirfile);
479 if fs_encoding != 'utf-8':
480 path = path.encode(fs_encoding).decode('utf-8');
481 retlist.append(path);
482 return retlist;
483
484 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
485 if isinstance(dirpath, (list, tuple, )):
486 dirpath = list(filter(None, dirpath));
487 elif isinstance(dirpath, str):
488 dirpath = list(filter(None, [dirpath]));
489 retlist = [];
490 fs_encoding = sys.getfilesystemencoding();
491 for mydirfile in dirpath:
492 if not os.path.exists(mydirfile):
493 return False;
494 mydirfile = NormalizeRelativePath(mydirfile);
495 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
496 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
497 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
498 for root, dirs, filenames in os.walk(mydirfile):
499 # Sort dirs and filenames alphabetically in place
500 dirs.sort(key=lambda x: x.lower());
501 filenames.sort(key=lambda x: x.lower());
502 dpath = RemoveWindowsPath(root);
503 if fs_encoding != 'utf-8':
504 dpath = dpath.encode(fs_encoding).decode('utf-8');
505 if not duplicates and dpath not in retlist:
506 retlist.append(dpath);
507 elif duplicates:
508 retlist.append(dpath);
509 for file in filenames:
510 fpath = os.path.join(root, file);
511 fpath = RemoveWindowsPath(fpath);
512 if fs_encoding != 'utf-8':
513 fpath = fpath.encode(fs_encoding).decode('utf-8');
514 if not duplicates and fpath not in retlist:
515 retlist.append(fpath);
516 elif duplicates:
517 retlist.append(fpath);
518 else:
519 path = RemoveWindowsPath(mydirfile);
520 if fs_encoding != 'utf-8':
521 path = path.encode(fs_encoding).decode('utf-8');
522 retlist.append(path);
523 return retlist;
524
525 def create_alias_function(prefix, base_name, suffix, target_function):
526 # Define a new function that wraps the target function
527 def alias_function(*args, **kwargs):
528 return target_function(*args, **kwargs);
529 # Create the function name by combining the prefix, base name, and the suffix
530 function_name = "{}{}{}".format(prefix, base_name, suffix);
531 # Add the new function to the global namespace
532 globals()[function_name] = alias_function;
533
534 def create_alias_function_alt(prefix, base_name, suffix, target_function):
535 # Create the function name by combining the prefix, base name, and the suffix
536 # Use the format method for string formatting, compatible with Python 2 and 3
537 function_name = "{}{}{}".format(prefix, base_name, suffix)
538 # Add the new function (alias of the target_function) to the global namespace
539 # This line is compatible as-is with both Python 2 and 3
540 globals()[function_name] = target_function
541
542 def FormatSpecsListToDict(formatspecs=__file_format_list__):
543 if(isinstance(formatspecs, (list, tuple, ))):
544 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
545 elif(isinstance(formatspecs, (dict, ))):
546 return formatspecs;
547 else:
548 return __file_format_dict__;
549 return __file_format_dict__;
550
551 class ZlibFile:
552 def __init__(self, file_path=None, fileobj=None, mode='rb', level=9, wbits=15, encoding=None, errors=None, newline=None):
553 if file_path is None and fileobj is None:
554 raise ValueError("Either file_path or fileobj must be provided");
555 if file_path is not None and fileobj is not None:
556 raise ValueError("Only one of file_path or fileobj should be provided");
557
558 self.file_path = file_path;
559 self.fileobj = fileobj;
560 self.mode = mode;
561 self.level = level;
562 self.wbits = wbits;
563 self.encoding = encoding;
564 self.errors = errors;
565 self.newline = newline;
566 self._compressed_data = b'';
567 self._decompressed_data = b'';
568 self._position = 0;
569 self._text_mode = 't' in mode;
570
571 # Force binary mode for internal handling
572 internal_mode = mode.replace('t', 'b');
573
574 if 'w' in mode or 'a' in mode or 'x' in mode:
575 self.file = open(file_path, internal_mode) if file_path else fileobj;
576 self._compressor = zlib.compressobj(level, zlib.DEFLATED, wbits);
577 elif 'r' in mode:
578 if file_path:
579 if os.path.exists(file_path):
580 self.file = open(file_path, internal_mode);
581 self._load_file();
582 else:
583 raise FileNotFoundError("No such file: '{}'".format(file_path));
584 elif fileobj:
585 self.file = fileobj;
586 self._load_file();
587 else:
588 raise ValueError("Mode should be 'rb' or 'wb'");
589
590 def _load_file(self):
591 self.file.seek(0);
592 self._compressed_data = self.file.read();
593 if not self._compressed_data.startswith((b'\x78\x01', b'\x78\x5E', b'\x78\x9C', b'\x78\xDA')):
594 raise ValueError("Invalid zlib file header");
595 self._decompressed_data = zlib.decompress(self._compressed_data, self.wbits);
596 if self._text_mode:
597 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
598
599 def write(self, data):
600 if self._text_mode:
601 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
602 compressed_data = self._compressor.compress(data) + self._compressor.flush(zlib.Z_SYNC_FLUSH);
603 self.file.write(compressed_data);
604
605 def read(self, size=-1):
606 if size == -1:
607 size = len(self._decompressed_data) - self._position;
608 data = self._decompressed_data[self._position:self._position + size];
609 self._position += size;
610 return data;
611
612 def seek(self, offset, whence=0):
613 if whence == 0: # absolute file positioning
614 self._position = offset;
615 elif whence == 1: # seek relative to the current position
616 self._position += offset;
617 elif whence == 2: # seek relative to the file's end
618 self._position = len(self._decompressed_data) + offset;
619 else:
620 raise ValueError("Invalid value for whence");
621
622 # Ensure the position is within bounds
623 self._position = max(0, min(self._position, len(self._decompressed_data)));
624
625 def tell(self):
626 return self._position;
627
628 def flush(self):
629 self.file.flush();
630
631 def fileno(self):
632 if hasattr(self.file, 'fileno'):
633 return self.file.fileno();
634 raise OSError("The underlying file object does not support fileno()");
635
636 def isatty(self):
637 if hasattr(self.file, 'isatty'):
638 return self.file.isatty();
639 return False;
640
641 def truncate(self, size=None):
642 if hasattr(self.file, 'truncate'):
643 return self.file.truncate(size);
644 raise OSError("The underlying file object does not support truncate()");
645
646 def close(self):
647 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
648 self.file.write(self._compressor.flush(zlib.Z_FINISH));
649 if self.file_path:
650 self.file.close();
651
652 def __enter__(self):
653 return self;
654
655 def __exit__(self, exc_type, exc_value, traceback):
656 self.close();
657
658 class GzipFile:
659 def __init__(self, file_path=None, fileobj=None, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None):
660 if file_path is None and fileobj is None:
661 raise ValueError("Either file_path or fileobj must be provided");
662 if file_path is not None and fileobj is not None:
663 raise ValueError("Only one of file_path or fileobj should be provided");
664
665 self.file_path = file_path;
666 self.fileobj = fileobj;
667 self.mode = mode;
668 self.compresslevel = compresslevel;
669 self.encoding = encoding;
670 self.errors = errors;
671 self.newline = newline;
672 self._compressed_data = b'';
673 self._decompressed_data = b'';
674 self._position = 0;
675 self._text_mode = 't' in mode;
676
677 # Force binary mode for internal handling
678 internal_mode = mode.replace('t', 'b');
679
680 if 'w' in mode or 'a' in mode or 'x' in mode:
681 self.file = gzip.open(file_path, internal_mode, compresslevel=compresslevel) if file_path else gzip.GzipFile(fileobj=fileobj, mode=internal_mode, compresslevel=compresslevel);
682 self._compressor = gzip.GzipFile(fileobj=self.file, mode=internal_mode, compresslevel=compresslevel);
683 elif 'r' in mode:
684 if file_path:
685 if os.path.exists(file_path):
686 self.file = gzip.open(file_path, internal_mode);
687 self._load_file();
688 else:
689 raise FileNotFoundError("No such file: '{}'".format(file_path));
690 elif fileobj:
691 self.file = gzip.GzipFile(fileobj=fileobj, mode=internal_mode);
692 self._load_file();
693 else:
694 raise ValueError("Mode should be 'rb' or 'wb'");
695
696 def _load_file(self):
697 self.file.seek(0);
698 self._compressed_data = self.file.read();
699 if not self._compressed_data.startswith(b'\x1f\x8b'):
700 raise ValueError("Invalid gzip file header");
701 self._decompressed_data = gzip.decompress(self._compressed_data);
702 if self._text_mode:
703 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
704
705 def write(self, data):
706 if self._text_mode:
707 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
708 compressed_data = self._compressor.compress(data);
709 self.file.write(compressed_data);
710 self.file.flush();
711
712 def read(self, size=-1):
713 if size == -1:
714 size = len(self._decompressed_data) - self._position;
715 data = self._decompressed_data[self._position:self._position + size];
716 self._position += size;
717 return data;
718
719 def seek(self, offset, whence=0):
720 if whence == 0: # absolute file positioning
721 self._position = offset;
722 elif whence == 1: # seek relative to the current position
723 self._position += offset;
724 elif whence == 2: # seek relative to the file's end
725 self._position = len(self._decompressed_data) + offset;
726 else:
727 raise ValueError("Invalid value for whence");
728
729 # Ensure the position is within bounds
730 self._position = max(0, min(self._position, len(self._decompressed_data)));
731
732 def tell(self):
733 return self._position;
734
735 def flush(self):
736 self.file.flush();
737
738 def fileno(self):
739 if hasattr(self.file, 'fileno'):
740 return self.file.fileno();
741 raise OSError("The underlying file object does not support fileno()");
742
743 def isatty(self):
744 if hasattr(self.file, 'isatty'):
745 return self.file.isatty();
746 return False;
747
748 def truncate(self, size=None):
749 if hasattr(self.file, 'truncate'):
750 return self.file.truncate(size);
751 raise OSError("The underlying file object does not support truncate()");
752
753 def close(self):
754 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
755 self.file.write(self._compressor.flush());
756 if self.file_path:
757 self.file.close();
758
759 def __enter__(self):
760 return self;
761
762 def __exit__(self, exc_type, exc_value, traceback):
763 self.close();
764
765 '''
766 class BloscFile:
767 def __init__(self, file_path=None, fileobj=None, mode='rb', level=9, encoding=None, errors=None, newline=None):
768 if file_path is None and fileobj is None:
769 raise ValueError("Either file_path or fileobj must be provided");
770 if file_path is not None and fileobj is not None:
771 raise ValueError("Only one of file_path or fileobj should be provided");
772
773 self.file_path = file_path;
774 self.fileobj = fileobj;
775 self.mode = mode;
776 self.level = level;
777 self.encoding = encoding;
778 self.errors = errors;
779 self.newline = newline;
780 self._compressed_data = b'';
781 self._decompressed_data = b'';
782 self._position = 0;
783 self._text_mode = 't' in mode;
784
785 # Force binary mode for internal handling
786 internal_mode = mode.replace('t', 'b');
787
788 if 'w' in mode or 'a' in mode or 'x' in mode:
789 self.file = open(file_path, internal_mode) if file_path else fileobj;
790 self._compressor = blosc.Blosc(level);
791 elif 'r' in mode:
792 if file_path:
793 if os.path.exists(file_path):
794 self.file = open(file_path, internal_mode);
795 self._load_file();
796 else:
797 raise FileNotFoundError("No such file: '{}'".format(file_path));
798 elif fileobj:
799 self.file = fileobj;
800 self._load_file();
801 else:
802 raise ValueError("Mode should be 'rb' or 'wb'");
803
804 def _load_file(self):
805 self.file.seek(0);
806 self._compressed_data = self.file.read();
807 if not self._compressed_data:
808 raise ValueError("Invalid blosc file header");
809 self._decompressed_data = blosc.decompress(self._compressed_data);
810 if self._text_mode:
811 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
812
813 def write(self, data):
814 if self._text_mode:
815 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
816 compressed_data = blosc.compress(data, cname='blosclz', clevel=self.level);
817 self.file.write(compressed_data);
818 self.file.flush();
819
820 def read(self, size=-1):
821 if size == -1:
822 size = len(self._decompressed_data) - self._position;
823 data = self._decompressed_data[self._position:self._position + size];
824 self._position += size;
825 return data;
826
827 def seek(self, offset, whence=0):
828 if whence == 0: # absolute file positioning
829 self._position = offset;
830 elif whence == 1: # seek relative to the current position
831 self._position += offset;
832 elif whence == 2: # seek relative to the file's end
833 self._position = len(self._decompressed_data) + offset;
834 else:
835 raise ValueError("Invalid value for whence");
836
837 # Ensure the position is within bounds
838 self._position = max(0, min(self._position, len(self._decompressed_data)));
839
840 def tell(self):
841 return self._position;
842
843 def flush(self):
844 self.file.flush();
845
846 def fileno(self):
847 if hasattr(self.file, 'fileno'):
848 return self.file.fileno();
849 raise OSError("The underlying file object does not support fileno()");
850
851 def isatty(self):
852 if hasattr(self.file, 'isatty'):
853 return self.file.isatty();
854 return False;
855
856 def truncate(self, size=None):
857 if hasattr(self.file, 'truncate'):
858 return self.file.truncate(size);
859 raise OSError("The underlying file object does not support truncate()");
860
861 def close(self):
862 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
863 self.file.write(blosc.compress(self._compressor.flush(), cname='blosclz', clevel=self.level));
864 if self.file_path:
865 self.file.close();
866
867 def __enter__(self):
868 return self;
869
870 def __exit__(self, exc_type, exc_value, traceback):
871 self.close();
872
873 class BrotliFile:
874 def __init__(self, file_path=None, fileobj=None, mode='rb', level=11, encoding=None, errors=None, newline=None):
875 if file_path is None and fileobj is None:
876 raise ValueError("Either file_path or fileobj must be provided");
877 if file_path is not None and fileobj is not None:
878 raise ValueError("Only one of file_path or fileobj should be provided");
879
880 self.file_path = file_path;
881 self.fileobj = fileobj;
882 self.mode = mode;
883 self.level = level;
884 self.encoding = encoding;
885 self.errors = errors;
886 self.newline = newline;
887 self._compressed_data = b'';
888 self._decompressed_data = b'';
889 self._position = 0;
890 self._text_mode = 't' in mode;
891
892 # Force binary mode for internal handling
893 internal_mode = mode.replace('t', 'b');
894
895 if 'w' in mode or 'a' in mode or 'x' in mode:
896 self.file = open(file_path, internal_mode) if file_path else fileobj;
897 self._compressor = brotli.Compressor(quality=self.level);
898 elif 'r' in mode:
899 if file_path:
900 if os.path.exists(file_path):
901 self.file = open(file_path, internal_mode);
902 self._load_file();
903 else:
904 raise FileNotFoundError("No such file: '{}'".format(file_path));
905 elif fileobj:
906 self.file = fileobj;
907 self._load_file();
908 else:
909 raise ValueError("Mode should be 'rb' or 'wb'");
910
911 def _load_file(self):
912 self.file.seek(0);
913 self._compressed_data = self.file.read();
914 if not self._compressed_data:
915 raise ValueError("Invalid brotli file header");
916 self._decompressed_data = brotli.decompress(self._compressed_data);
917 if self._text_mode:
918 self._decompressed_data = self._decompressed_data.decode(self.encoding or 'utf-8', self.errors or 'strict');
919
920 def write(self, data):
921 if self._text_mode:
922 data = data.encode(self.encoding or 'utf-8', self.errors or 'strict');
923 compressed_data = self._compressor.process(data);
924 self.file.write(compressed_data);
925 self.file.flush();
926
927 def read(self, size=-1):
928 if size == -1:
929 size = len(self._decompressed_data) - self._position;
930 data = self._decompressed_data[self._position:self._position + size];
931 self._position += size;
932 return data;
933
934 def seek(self, offset, whence=0):
935 if whence == 0: # absolute file positioning
936 self._position = offset;
937 elif whence == 1: # seek relative to the current position
938 self._position += offset;
939 elif whence == 2: # seek relative to the file's end
940 self._position = len(self._decompressed_data) + offset;
941 else:
942 raise ValueError("Invalid value for whence");
943
944 # Ensure the position is within bounds
945 self._position = max(0, min(self._position, len(self._decompressed_data)));
946
947 def tell(self):
948 return self._position;
949
950 def flush(self):
951 self.file.flush();
952
953 def fileno(self):
954 if hasattr(self.file, 'fileno'):
955 return self.file.fileno();
956 raise OSError("The underlying file object does not support fileno()");
957
958 def isatty(self):
959 if hasattr(self.file, 'isatty'):
960 return self.file.isatty();
961 return False;
962
963 def truncate(self, size=None):
964 if hasattr(self.file, 'truncate'):
965 return self.file.truncate(size);
966 raise OSError("The underlying file object does not support truncate()");
967
968 def close(self):
969 if 'w' in self.mode or 'a' in self.mode or 'x' in self.mode:
970 self.file.write(self._compressor.finish());
971 if self.file_path:
972 self.file.close();
973
974 def __enter__(self):
975 return self;
976
977 def __exit__(self, exc_type, exc_value, traceback):
978 self.close();
979 '''
980
981 def TarFileCheck(infile):
982 try:
983 if is_tarfile(infile):
984 return True;
985 else:
986 return False;
987 except TypeError:
988 try:
989 # Check if the input is a file object
990 if hasattr(infile, 'read'):
991 # Save the current file position
992 current_position = infile.tell();
993 # Attempt to open the file object as a tar file
994 tar = tarfile.open(fileobj=infile);
995 tar.close();
996 # Restore the file position
997 infile.seek(current_position);
998 return True;
999 else:
1000 # Assume it's a filename
1001 tar = tarfile.open(name=infile);
1002 tar.close();
1003 return True;
1004 except tarfile.TarError:
1005 return False;
1006
1007 def TarFileCheckAlt(infile):
1008 try:
1009 if is_tarfile(infile):
1010 return True;
1011 except TypeError:
1012 pass;
1013 try:
1014 # Check if the input is a file-like object
1015 if hasattr(infile, 'read'):
1016 # Save the current file position
1017 current_position = infile.tell();
1018 # Attempt to open the file object as a tar file
1019 with tarfile.open(fileobj=infile) as tar:
1020 pass;
1021 # Restore the file position
1022 infile.seek(current_position);
1023 else:
1024 # Assume it's a filename and attempt to open it as a tar file
1025 with tarfile.open(name=infile) as tar:
1026 pass;
1027 return True;
1028 except (tarfile.TarError, AttributeError, IOError):
1029 return False;
1030
1031 def ZipFileCheck(infile):
1032 try:
1033 if zipfile.is_zipfile(infile):
1034 return True;
1035 except TypeError:
1036 pass;
1037 try:
1038 # Check if the input is a file-like object
1039 if hasattr(infile, 'read'):
1040 # Save the current file position
1041 current_position = infile.tell();
1042 # Attempt to open the file object as a zip file
1043 with zipfile.ZipFile(infile) as zipf:
1044 pass;
1045 # Restore the file position
1046 infile.seek(current_position);
1047 else:
1048 # Assume it's a filename and attempt to open it as a zip file
1049 with zipfile.ZipFile(infile) as zipf:
1050 pass;
1051 return True;
1052 except (zipfile.BadZipFile, AttributeError, IOError):
1053 return False;
1054
1055 def RarFileCheck(infile):
1056 try:
1057 if rarfile.is_rarfile(infile):
1058 return True;
1059 except TypeError:
1060 pass;
1061 try:
1062 # Check if the input is a file-like object
1063 if hasattr(infile, 'read'):
1064 # Save the current file position
1065 current_position = infile.tell();
1066 # Attempt to open the file object as a rar file
1067 with rarfile.RarFile(infile) as rarf:
1068 pass;
1069 # Restore the file position
1070 infile.seek(current_position);
1071 else:
1072 # Assume it's a filename and attempt to open it as a rar file
1073 with rarfile.RarFile(infile) as rarf:
1074 pass;
1075 return True;
1076 except (rarfile.Error, AttributeError, IOError):
1077 return False;
1078
1079 def SevenZipFileCheck(infile):
1080 try:
1081 # Check if the input is a file-like object
1082 if hasattr(infile, 'read'):
1083 # Save the current file position
1084 current_position = infile.tell();
1085 # Attempt to open the file object as a 7z file
1086 with py7zr.SevenZipFile(infile, 'r') as archive:
1087 pass;
1088 # Restore the file position
1089 infile.seek(current_position);
1090 else:
1091 # Assume it's a filename and attempt to open it as a 7z file
1092 with py7zr.SevenZipFile(infile, 'r') as archive:
1093 pass;
1094 return True;
1095 except (py7zr.Bad7zFile, AttributeError, IOError):
1096 return False;
1097
1098 # initial_value can be 0xFFFF or 0x0000
1099 def crc16_ansi(msg, initial_value=0xFFFF):
1100 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
1101 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
1102 crc = initial_value; # Initial value
1103 for b in msg:
1104 crc ^= b << 8; # XOR byte into CRC top byte
1105 for _ in range(8): # Process each bit
1106 if crc & 0x8000: # If the top bit is set
1107 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
1108 else:
1109 crc = crc << 1; # Just shift left
1110 crc &= 0xFFFF; # Ensure CRC remains 16-bit
1111 return crc;
1112
1113 # initial_value can be 0xFFFF or 0x0000
1114 def crc16_ibm(msg, initial_value=0xFFFF):
1115 return crc16_ansi(msg, initial_value);
1116
1117 # initial_value is 0xFFFF
1118 def crc16(msg):
1119 return crc16_ansi(msg, 0xFFFF);
1120
1121 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
1122 def crc16_ccitt(msg, initial_value=0xFFFF):
1123 # CRC-16-CCITT polynomial
1124 poly = 0x1021; # Polynomial for CRC-16-CCITT
1125 # Use the specified initial value
1126 crc = initial_value;
1127 for b in msg:
1128 crc ^= b << 8; # XOR byte into CRC top byte
1129 for _ in range(8): # Process each bit
1130 if crc & 0x8000: # If the top bit is set
1131 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
1132 else:
1133 crc = crc << 1; # Just shift left
1134 crc &= 0xFFFF; # Ensure CRC remains 16-bit
1135 return crc;
1136
1137 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
1138 def crc64_ecma(msg, initial_value=0x0000000000000000):
1139 # CRC-64-ECMA polynomial and initial value
1140 poly = 0x42F0E1EBA9EA3693;
1141 crc = initial_value; # Initial value for CRC-64-ECMA
1142 for b in msg:
1143 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
1144 for _ in range(8): # Process each bit
1145 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
1146 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
1147 else:
1148 crc <<= 1; # Just shift left if the MSB is 0
1149 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
1150 return crc;
1151
1152 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
1153 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
1154 # CRC-64-ISO polynomial and initial value
1155 poly = 0x000000000000001B;
1156 crc = initial_value; # Common initial value for CRC-64-ISO
1157 for b in msg:
1158 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
1159 for _ in range(8): # Process each bit
1160 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
1161 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
1162 else:
1163 crc <<= 1; # Just shift left if the MSB is 0
1164 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
1165 return crc;
1166
1167 def GetDataFromArray(data, path, default=None):
1168 element = data;
1169 try:
1170 for key in path:
1171 element = element[key];
1172 return element;
1173 except (KeyError, TypeError, IndexError):
1174 return default;
1175
1176 def GetDataFromArrayAlt(structure, path, default=None):
1177 element = structure;
1178 for key in path:
1179 if isinstance(element, dict) and key in element:
1180 element = element[key];
1181 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
1182 element = element[key];
1183 else:
1184 return default;
1185 return element;
1186
1187 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
1188 formatspecs = FormatSpecsListToDict(formatspecs);
1189 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
1190 if encodedata:
1191 fileheader = fileheader.encode('UTF-8');
1192 checksum_methods = {
1193 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1194 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1195 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1196 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
1197 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
1198 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
1199 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
1200 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1201 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1202 };
1203 if checksumtype in checksum_methods:
1204 return checksum_methods[checksumtype](fileheader);
1205 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
1206 checksumoutstr = hashlib.new(checksumtype);
1207 checksumoutstr.update(fileheader);
1208 return checksumoutstr.hexdigest().lower();
1209 return format(0, 'x').lower();
1210
1211 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
1212 formatspecs = FormatSpecsListToDict(formatspecs);
1213 if encodedata:
1214 instr = instr.encode('UTF-8');
1215 checksum_methods = {
1216 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1217 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1218 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
1219 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
1220 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
1221 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
1222 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
1223 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1224 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
1225 };
1226 if checksumtype in checksum_methods:
1227 return checksum_methods[checksumtype](instr);
1228 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
1229 checksumoutstr = hashlib.new(checksumtype);
1230 checksumoutstr.update(instr);
1231 return checksumoutstr.hexdigest().lower();
1232 return format(0, 'x').lower();
1233
1234 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
1235 formatspecs = FormatSpecsListToDict(formatspecs);
1236 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
1237 return inchecksum.lower() == catfileheadercshex;
1238
1239 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
1240 formatspecs = FormatSpecsListToDict(formatspecs);
1241 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
1242 return inchecksum.lower() == catinfilecshex;
1243
1244 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
1245 curbyte = b"";
1246 curfullbyte = b"";
1247 nullbyte = delimiter.encode("UTF-8");
1248 while(True):
1249 curbyte = fp.read(1);
1250 if(curbyte==nullbyte or not curbyte):
1251 break;
1252 curfullbyte = curfullbyte + curbyte;
1253 return curfullbyte.decode('UTF-8');
1254
1255 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
1256 return ReadTillNullByteOld(fp, delimiter);
1257
1258 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
1259 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
1260 buffer = bytearray();
1261 total_read = 0;
1262 delimiter_length = len(delimiter);
1263 while True:
1264 chunk = fp.read(chunk_size)
1265 if not chunk:
1266 # End of file reached without finding the delimiter
1267 break;
1268 buffer.extend(chunk);
1269 total_read += len(chunk);
1270 if delimiter in buffer:
1271 # Delimiter found, calculate where to reset the file pointer
1272 index = buffer.find(delimiter);
1273 # Calculate how many extra bytes were read after the delimiter
1274 extra_bytes_read = len(buffer) - (index + delimiter_length);
1275 # Move the file pointer back to just after the delimiter
1276 fp.seek(-extra_bytes_read, 1);
1277 buffer = buffer[:index];
1278 break;
1279 if total_read >= max_read:
1280 # Stop reading if max limit is reached to prevent excessive memory usage
1281 raise MemoryError("Maximum read limit reached without finding the delimiter.");
1282 # Check for incomplete UTF-8 sequences at the end of the buffer
1283 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
1284 # This suggests that the last byte might be the start of a multi-byte character
1285 # Try to read one more byte to complete the character
1286 extra_byte = fp.read(1);
1287 if extra_byte:
1288 buffer.extend(extra_byte);
1289 else:
1290 # No more data available
1291 break;
1292 try:
1293 return buffer.decode('UTF-8', errors='replace');
1294 except UnicodeDecodeError:
1295 return buffer.decode('UTF-8', errors='replace');
1296
1297 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
1298 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
1299
1300 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
1301 curfullbyte = bytearray();
1302 nullbyte = delimiter.encode("UTF-8");
1303 total_read = 0; # Track the total number of bytes read
1304 while True:
1305 curbyte = fp.read(1);
1306 if curbyte == nullbyte or not curbyte:
1307 break;
1308 curfullbyte.extend(curbyte);
1309 total_read += 1;
1310 if total_read >= max_read:
1311 raise MemoryError("Maximum read limit reached without finding the delimiter.");
1312 # Decode the full byte array to string once out of the loop
1313 try:
1314 return curfullbyte.decode('UTF-8');
1315 except UnicodeDecodeError:
1316 # Handle potential partial UTF-8 characters
1317 for i in range(1, 4):
1318 try:
1319 return curfullbyte[:-i].decode('UTF-8');
1320 except UnicodeDecodeError:
1321 continue;
1322 raise; # Re-raise if decoding fails even after trimming
1323
1324 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
1325 return ReadTillNullByte(fp, delimiter, max_read);
1326
1327 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
1328 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
1329 buffer = bytearray();
1330 total_read = 0;
1331 delimiter_length = len(delimiter);
1332 results = [];
1333 while len(results) < num_delimiters:
1334 chunk = fp.read(chunk_size);
1335 if not chunk:
1336 # End of file reached; decode whatever is collected if it's the last needed part
1337 if len(buffer) > 0:
1338 results.append(buffer.decode('UTF-8', errors='replace'));
1339 break;
1340 buffer.extend(chunk)
1341 total_read += len(chunk);
1342 # Check if we have found the delimiter
1343 while delimiter in buffer:
1344 index = buffer.find(delimiter);
1345 # Decode the section before the delimiter
1346 results.append(buffer[:index].decode('UTF-8', errors='replace'));
1347 # Remove the processed part from the buffer
1348 buffer = buffer[index + delimiter_length:];
1349 if len(results) == num_delimiters:
1350 # If reached the required number of delimiters, adjust the file pointer and stop
1351 fp.seek(-len(buffer), 1);
1352 return results;
1353 if total_read >= max_read:
1354 # Stop reading if max limit is reached to prevent excessive memory usage
1355 raise MemoryError("Maximum read limit reached without finding the delimiter.");
1356 # Check for incomplete UTF-8 sequences at the end of the buffer
1357 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
1358 # This suggests that the last byte might be the start of a multi-byte character
1359 # Try to read one more byte to complete the character
1360 extra_byte = fp.read(1);
1361 if extra_byte:
1362 buffer.extend(extra_byte);
1363 else:
1364 # No more data available
1365 break;
1366 # Process remaining buffer if less than the required number of delimiters were found
1367 if len(buffer) > 0 and len(results) < num_delimiters:
1368 results.append(buffer.decode('UTF-8', errors='replace'));
1369 return results;
1370
1371 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
1372 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
1373
1374 def SeekToEndOfFile(fp):
1375 lasttell = 0;
1376 while(True):
1377 fp.seek(1, 1);
1378 if(lasttell==fp.tell()):
1379 break;
1380 lasttell = fp.tell();
1381 return True;
1382
1383 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
1384 rocount = 0;
1385 roend = int(rounds);
1386 HeaderOut = [];
1387 while(rocount<roend):
1388 HeaderOut.append(ReadTillNullByte(fp, delimiter));
1389 rocount = rocount + 1;
1390 return HeaderOut;
1391
1392 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
1393 headerpresize = ReadTillNullByte(fp, delimiter);
1394 headersize = int(headerpresize, 16);
1395 if(headersize<=0):
1396 return [];
1397 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
1398 fp.seek(1, 1);
1399 rocount = 0;
1400 roend = int(len(headercontent));
1401 HeaderOut = [headerpresize];
1402 while(rocount<roend):
1403 HeaderOut.append(headercontent[rocount]);
1404 rocount = rocount + 1;
1405 return HeaderOut;
1406
1407 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
1408 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
1409 headersize = int(preheaderdata[0], 16);
1410 headernumfields = int(preheaderdata[1], 16);
1411 if(headersize<=0 or headernumfields<=0):
1412 return [];
1413 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
1414 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
1415 HeaderOut = preheaderdata + headerdata;
1416 return HeaderOut;
1417
1418 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1419 formatspecs = FormatSpecsListToDict(formatspecs);
1420 delimiter = formatspecs['format_delimiter'];
1421 fheaderstart = fp.tell();
1422 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1423 if(len(HeaderOut)==0):
1424 return False;
1425 if(re.findall(r"^[.|/]", HeaderOut[3])):
1426 fname = HeaderOut[3];
1427 else:
1428 fname = "./"+HeaderOut[3];
1429 fcs = HeaderOut[-2].lower();
1430 fccs = HeaderOut[-1].lower();
1431 fsize = int(HeaderOut[5], 16);
1432 fcompression = HeaderOut[12];
1433 fcsize = int(HeaderOut[13], 16);
1434 fseeknextfile = HeaderOut[25];
1435 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1436 if(fcs!=newfcs and not skipchecksum):
1437 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1438 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1439 return False;
1440 fhend = fp.tell() - 1;
1441 fcontentstart = fp.tell();
1442 fcontents = BytesIO();
1443 if(fsize>0 and not listonly):
1444 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1445 fcontents.write(fp.read(fsize));
1446 else:
1447 fcontents.write(fp.read(fcsize));
1448 elif(fsize>0 and listonly):
1449 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1450 fp.seek(fsize, 1);
1451 else:
1452 fp.seek(fcsize, 1);
1453 fcontents.seek(0, 0);
1454 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1455 if(fccs!=newfccs and not skipchecksum and not listonly):
1456 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1457 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1458 return False;
1459 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1460 pass;
1461 else:
1462 fcontents.seek(0, 0);
1463 if(uncompress):
1464 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1465 fcontentend = fp.tell();
1466 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1467 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1468 if(abs(fseeknextasnum)==0):
1469 pass;
1470 fp.seek(fseeknextasnum, 1);
1471 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1472 fseeknextasnum = int(fseeknextfile);
1473 if(abs(fseeknextasnum)==0):
1474 pass;
1475 fp.seek(fseeknextasnum, 1);
1476 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1477 fseeknextasnum = int(fseeknextfile);
1478 if(abs(fseeknextasnum)==0):
1479 pass;
1480 fp.seek(fseeknextasnum, 0);
1481 else:
1482 return False;
1483 HeaderOut.append(fcontents);
1484 return HeaderOut;
1485
1486 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, contentasfile=True, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1487 formatspecs = FormatSpecsListToDict(formatspecs);
1488 delimiter = formatspecs['format_delimiter'];
1489 fheaderstart = fp.tell();
1490 if(formatspecs['new_style']):
1491 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1492 else:
1493 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1494 if(len(HeaderOut)==0):
1495 return False;
1496 fheadsize = int(HeaderOut[0], 16);
1497 fnumfields = int(HeaderOut[1], 16);
1498 ftype = int(HeaderOut[2], 16);
1499 if(re.findall(r"^[.|/]", HeaderOut[3])):
1500 fname = HeaderOut[3];
1501 else:
1502 fname = "./"+HeaderOut[3];
1503 fbasedir = os.path.dirname(fname);
1504 flinkname = HeaderOut[4];
1505 fsize = int(HeaderOut[5], 16);
1506 fatime = int(HeaderOut[6], 16);
1507 fmtime = int(HeaderOut[7], 16);
1508 fctime = int(HeaderOut[8], 16);
1509 fbtime = int(HeaderOut[9], 16);
1510 fmode = int(HeaderOut[10], 16);
1511 fchmode = stat.S_IMODE(fmode);
1512 ftypemod = stat.S_IFMT(fmode);
1513 fwinattributes = int(HeaderOut[11], 16);
1514 fcompression = HeaderOut[12];
1515 fcsize = int(HeaderOut[13], 16);
1516 fuid = int(HeaderOut[14], 16);
1517 funame = HeaderOut[15];
1518 fgid = int(HeaderOut[16], 16);
1519 fgname = HeaderOut[17];
1520 fid = int(HeaderOut[18], 16);
1521 finode = int(HeaderOut[19], 16);
1522 flinkcount = int(HeaderOut[20], 16);
1523 fdev_minor = int(HeaderOut[21], 16);
1524 fdev_major = int(HeaderOut[22], 16);
1525 frdev_minor = int(HeaderOut[23], 16);
1526 frdev_major = int(HeaderOut[24], 16);
1527 fseeknextfile = HeaderOut[25];
1528 fextrasize = int(HeaderOut[26], 16);
1529 fextrafields = int(HeaderOut[27], 16);
1530 extrafieldslist = [];
1531 extrastart = 28
1532 extraend = extrastart + fextrafields;
1533 extrafieldslist = [];
1534 if(extrastart<extraend):
1535 extrafieldslist.append(HeaderOut[extrastart]);
1536 extrastart = extrastart + 1;
1537 fcs = HeaderOut[-2].lower();
1538 fccs = HeaderOut[-1].lower();
1539 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1540 if(fcs!=newfcs and not skipchecksum):
1541 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1542 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1543 return False;
1544 fhend = fp.tell() - 1;
1545 fcontentstart = fp.tell();
1546 fcontents = BytesIO();
1547 pyhascontents = False;
1548 if(fsize>0 and not listonly):
1549 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1550 fcontents.write(fp.read(fsize));
1551 else:
1552 fcontents.write(fp.read(fcsize));
1553 pyhascontents = True;
1554 elif(fsize>0 and listonly):
1555 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1556 fp.seek(fsize, 1);
1557 else:
1558 fp.seek(fcsize, 1);
1559 pyhascontents = False;
1560 fcontents.seek(0, 0);
1561 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1562 if(fccs!=newfccs and not skipchecksum and not listonly):
1563 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1564 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1565 return False;
1566 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1567 pass;
1568 else:
1569 fcontents.seek(0, 0);
1570 if(uncompress):
1571 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1572 fcontents.seek(0, 0);
1573 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1574 fcontentend = fp.tell() - 1;
1575 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1576 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1577 if(abs(fseeknextasnum)==0):
1578 pass;
1579 fp.seek(fseeknextasnum, 1);
1580 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1581 fseeknextasnum = int(fseeknextfile);
1582 if(abs(fseeknextasnum)==0):
1583 pass;
1584 fp.seek(fseeknextasnum, 1);
1585 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1586 fseeknextasnum = int(fseeknextfile);
1587 if(abs(fseeknextasnum)==0):
1588 pass;
1589 fp.seek(fseeknextasnum, 0);
1590 else:
1591 return False;
1592 fcontents.seek(0, 0);
1593 if(not contentasfile):
1594 fcontents = fcontents.read();
1595 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontentasfile': contentasfile, 'fcontents': fcontents};
1596 return catlist;
1597
1598 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1599 formatspecs = FormatSpecsListToDict(formatspecs);
1600 delimiter = formatspecs['format_delimiter'];
1601 fheaderstart = fp.tell();
1602 if(formatspecs['new_style']):
1603 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1604 else:
1605 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1606 if(len(HeaderOut)==0):
1607 return False;
1608 fheadsize = int(HeaderOut[0], 16);
1609 fnumfields = int(HeaderOut[1], 16);
1610 ftype = int(HeaderOut[2], 16);
1611 if(re.findall(r"^[.|/]", HeaderOut[3])):
1612 fname = HeaderOut[3];
1613 else:
1614 fname = "./"+HeaderOut[3];
1615 fbasedir = os.path.dirname(fname);
1616 flinkname = HeaderOut[4];
1617 fsize = int(HeaderOut[5], 16);
1618 fatime = int(HeaderOut[6], 16);
1619 fmtime = int(HeaderOut[7], 16);
1620 fctime = int(HeaderOut[8], 16);
1621 fbtime = int(HeaderOut[9], 16);
1622 fmode = int(HeaderOut[10], 16);
1623 fchmode = stat.S_IMODE(fmode);
1624 ftypemod = stat.S_IFMT(fmode);
1625 fwinattributes = int(HeaderOut[11], 16);
1626 fcompression = HeaderOut[12];
1627 fcsize = int(HeaderOut[13], 16);
1628 fuid = int(HeaderOut[14], 16);
1629 funame = HeaderOut[15];
1630 fgid = int(HeaderOut[16], 16);
1631 fgname = HeaderOut[17];
1632 fid = int(HeaderOut[18], 16);
1633 finode = int(HeaderOut[19], 16);
1634 flinkcount = int(HeaderOut[20], 16);
1635 fdev_minor = int(HeaderOut[21], 16);
1636 fdev_major = int(HeaderOut[22], 16);
1637 frdev_minor = int(HeaderOut[23], 16);
1638 frdev_major = int(HeaderOut[24], 16);
1639 fseeknextfile = HeaderOut[25];
1640 fextrasize = int(HeaderOut[26], 16);
1641 fextrafields = int(HeaderOut[27], 16);
1642 extrafieldslist = [];
1643 extrastart = 28;
1644 extraend = extrastart + fextrafields;
1645 extrafieldslist = [];
1646 if(extrastart<extraend):
1647 extrafieldslist.append(HeaderOut[extrastart]);
1648 extrastart = extrastart + 1;
1649 fheaderchecksumtype = HeaderOut[extrastart].lower();
1650 fcontentchecksumtype = HeaderOut[extrastart + 1].lower();
1651 fcs = HeaderOut[-2].lower();
1652 fccs = HeaderOut[-1].lower();
1653 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1654 if(fcs!=newfcs and not skipchecksum):
1655 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1656 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1657 return False;
1658 fhend = fp.tell() - 1;
1659 fcontentstart = fp.tell();
1660 fcontents = BytesIO();
1661 pyhascontents = False;
1662 if(fsize>0 and not listonly):
1663 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1664 fcontents.write(fp.read(fsize));
1665 else:
1666 fcontents.write(fp.read(fcsize));
1667 pyhascontents = True;
1668 elif(fsize>0 and listonly):
1669 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1670 fp.seek(fsize, 1);
1671 else:
1672 fp.seek(fcsize, 1);
1673 pyhascontents = False;
1674 fcontents.seek(0, 0);
1675 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1676 if(fccs!=newfccs and not skipchecksum and not listonly):
1677 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1678 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1679 return False;
1680 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1681 pass;
1682 else:
1683 fcontents.seek(0, 0);
1684 if(uncompress):
1685 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1686 fcontents.seek(0, 0);
1687 fcontentend = fp.tell() - 1;
1688 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1689 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1690 if(abs(fseeknextasnum)==0):
1691 pass;
1692 fp.seek(fseeknextasnum, 1);
1693 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1694 fseeknextasnum = int(fseeknextfile);
1695 if(abs(fseeknextasnum)==0):
1696 pass;
1697 fp.seek(fseeknextasnum, 1);
1698 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1699 fseeknextasnum = int(fseeknextfile);
1700 if(abs(fseeknextasnum)==0):
1701 pass;
1702 fp.seek(fseeknextasnum, 0);
1703 else:
1704 return False;
1705 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1706 return catlist;
1707
1708 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1709 formatspecs = FormatSpecsListToDict(formatspecs);
1710 delimiter = formatspecs['format_delimiter'];
1711 curloc = fp.tell();
1712 if(curloc>0):
1713 fp.seek(0, 0);
1714 catheader = ReadFileHeaderData(fp, 4, delimiter);
1715 if(curloc>0):
1716 fp.seek(curloc, 0);
1717 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1718 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1719 if(not headercheck and not skipchecksum):
1720 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1721 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1722 return False;
1723 fnumfiles = int(catheader[1], 16);
1724 countnum = 0;
1725 flist = [];
1726 while(countnum < fnumfiles):
1727 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1728 if(len(HeaderOut)==0):
1729 break;
1730 flist.append(HeaderOut);
1731 countnum = countnum + 1;
1732 return flist;
1733
1734 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, contentasfile=True, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1735 formatspecs = FormatSpecsListToDict(formatspecs);
1736 delimiter = formatspecs['format_delimiter'];
1737 curloc = fp.tell();
1738 if(curloc>0):
1739 fp.seek(0, 0);
1740 catheader = ReadFileHeaderData(fp, 4, delimiter);
1741 if(curloc>0):
1742 fp.seek(curloc, 0);
1743 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1744 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1745 if(not headercheck and not skipchecksum):
1746 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1747 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1748 return False;
1749 catstring = catheader[0];
1750 catversion = re.findall(r"([\d]+)", catstring);
1751 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1752 fprenumfiles = catheader[1];
1753 fnumfiles = int(fprenumfiles, 16);
1754 fprechecksumtype = catheader[2];
1755 fprechecksum = catheader[3];
1756 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1757 if(seekstart<0 and seekstart>fnumfiles):
1758 seekstart = 0;
1759 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1760 seekend = fnumfiles;
1761 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1762 seekend = fnumfiles - abs(seekend);
1763 if(seekstart>0):
1764 il = 0;
1765 while(il < seekstart):
1766 prefhstart = fp.tell();
1767 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1768 if(len(preheaderdata)==0):
1769 break;
1770 prefsize = int(preheaderdata[5], 16);
1771 prefseeknextfile = preheaderdata[25];
1772 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1773 prefcs = preheaderdata[-2];
1774 if(prefcs!=prenewfcs and not skipchecksum):
1775 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1776 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1777 return False;
1778 valid_archive = False;
1779 invalid_archive = True;
1780 prefhend = fp.tell() - 1;
1781 prefcontentstart = fp.tell();
1782 prefcontents = BytesIO();
1783 pyhascontents = False;
1784 if(prefsize>0):
1785 prefcontents.write(fp.read(prefsize));
1786 prefcontents.seek(0, 0);
1787 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1788 prefccs = preheaderdata[-1];
1789 pyhascontents = True;
1790 if(prefccs!=prenewfccs and not skipchecksum):
1791 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1792 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1793 return False;
1794 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1795 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1796 if(abs(fseeknextasnum)==0):
1797 pass;
1798 fp.seek(fseeknextasnum, 1);
1799 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1800 fseeknextasnum = int(prefseeknextfile);
1801 if(abs(fseeknextasnum)==0):
1802 pass;
1803 fp.seek(fseeknextasnum, 1);
1804 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1805 fseeknextasnum = int(prefseeknextfile);
1806 if(abs(fseeknextasnum)==0):
1807 pass;
1808 fp.seek(fseeknextasnum, 0);
1809 else:
1810 return False;
1811 il = il + 1;
1812 realidnum = 0;
1813 countnum = seekstart;
1814 while(countnum < seekend):
1815 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, contentasfile, uncompress, skipchecksum, formatspecs);
1816 if(len(HeaderOut)==0):
1817 break;
1818 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1819 catlist['ffilelist'].append(HeaderOut);
1820 countnum = countnum + 1;
1821 realidnum = realidnum + 1;
1822 return catlist;
1823
1824 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1825 formatspecs = FormatSpecsListToDict(formatspecs);
1826 delimiter = formatspecs['format_delimiter'];
1827 curloc = fp.tell();
1828 if(curloc>0):
1829 fp.seek(0, 0);
1830 catheader = ReadFileHeaderData(fp, 4, delimiter);
1831 if(curloc>0):
1832 fp.seek(curloc, 0);
1833 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1834 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1835 if(not headercheck and not skipchecksum):
1836 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1837 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1838 return False;
1839 catstring = catheader[0];
1840 catversion = re.findall(r"([\d]+)", catstring);
1841 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1842 fprenumfiles = catheader[1];
1843 fnumfiles = int(fprenumfiles, 16);
1844 fprechecksumtype = catheader[2];
1845 fprechecksum = catheader[3];
1846 catlist = [];
1847 if(seekstart<0 and seekstart>fnumfiles):
1848 seekstart = 0;
1849 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1850 seekend = fnumfiles;
1851 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1852 seekend = fnumfiles - abs(seekend);
1853 if(seekstart>0):
1854 il = 0;
1855 while(il < seekstart):
1856 prefhstart = fp.tell();
1857 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1858 if(len(preheaderdata)==0):
1859 break;
1860 prefsize = int(preheaderdata[5], 16);
1861 prefcompression = preheaderdata[12];
1862 prefcsize = int(preheaderdata[13], 16);
1863 prefseeknextfile = HeaderOut[25];
1864 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1865 prefcs = preheaderdata[-2];
1866 if(prefcs!=prenewfcs and not skipchecksum):
1867 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1868 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1869 return False;
1870 valid_archive = False;
1871 invalid_archive = True;
1872 prefhend = fp.tell() - 1;
1873 prefcontentstart = fp.tell();
1874 prefcontents = "";
1875 pyhascontents = False;
1876 if(prefsize>0):
1877 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1878 prefcontents = catfp.read(prefsize);
1879 else:
1880 prefcontents = catfp.read(prefcsize);
1881 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1882 prefccs = preheaderdata[-1];
1883 pyhascontents = True;
1884 if(prefccs!=prenewfccs and not skipchecksum):
1885 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1886 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1887 return False;
1888 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1889 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1890 if(abs(fseeknextasnum)==0):
1891 pass;
1892 catfp.seek(fseeknextasnum, 1);
1893 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1894 fseeknextasnum = int(prefseeknextfile);
1895 if(abs(fseeknextasnum)==0):
1896 pass;
1897 catfp.seek(fseeknextasnum, 1);
1898 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1899 fseeknextasnum = int(prefseeknextfile);
1900 if(abs(fseeknextasnum)==0):
1901 pass;
1902 catfp.seek(fseeknextasnum, 0);
1903 else:
1904 return False;
1905 il = il + 1;
1906 realidnum = 0;
1907 countnum = seekstart;
1908 while(countnum < seekend):
1909 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1910 if(len(HeaderOut)==0):
1911 break;
1912 catlist.append(HeaderOut);
1913 countnum = countnum + 1;
1914 realidnum = realidnum + 1;
1915 return catlist;
1916
1917 def ReadInFileBySizeWithContentToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1918 formatspecs = FormatSpecsListToDict(formatspecs);
1919 delimiter = formatspecs['format_delimiter'];
1920 if(hasattr(infile, "read") or hasattr(infile, "write")):
1921 fp = infile;
1922 fp.seek(0, 0);
1923 fp = UncompressArchiveFile(fp, formatspecs);
1924 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1925 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1926 return False;
1927 if(not fp):
1928 return False;
1929 fp.seek(0, 0);
1930 elif(infile=="-"):
1931 fp = BytesIO();
1932 if(hasattr(sys.stdin, "buffer")):
1933 shutil.copyfileobj(sys.stdin.buffer, fp);
1934 else:
1935 shutil.copyfileobj(sys.stdin, fp);
1936 fp.seek(0, 0);
1937 fp = UncompressArchiveFile(fp, formatspecs);
1938 if(not fp):
1939 return False;
1940 fp.seek(0, 0);
1941 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1942 fp = download_file_from_internet_file(infile);
1943 fp = UncompressArchiveFile(fp, formatspecs);
1944 fp.seek(0, 0);
1945 if(not fp):
1946 return False;
1947 fp.seek(0, 0);
1948 else:
1949 infile = RemoveWindowsPath(infile);
1950 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1951 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1952 return False;
1953 compresscheck = CheckCompressionType(infile, formatspecs, True);
1954 if(not compresscheck):
1955 fextname = os.path.splitext(infile)[1];
1956 if(fextname==".gz"):
1957 compresscheck = "gzip";
1958 elif(fextname==".bz2"):
1959 compresscheck = "bzip2";
1960 elif(fextname==".zst"):
1961 compresscheck = "zstd";
1962 elif(fextname==".lz4" or fextname==".clz4"):
1963 compresscheck = "lz4";
1964 elif(fextname==".lzo" or fextname==".lzop"):
1965 compresscheck = "lzo";
1966 elif(fextname==".lzma"):
1967 compresscheck = "lzma";
1968 elif(fextname==".xz"):
1969 compresscheck = "xz";
1970 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
1971 compresscheck = "zlib";
1972 else:
1973 return False;
1974 if(not compresscheck):
1975 return False;
1976 fp = UncompressFile(infile, formatspecs, "rb");
1977 return ReadFileDataBySizeWithContentToArray(fp, seekstart, seekend, listonly, contentasfile, uncompress, skipchecksum, formatspecs);
1978
1979 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1980 formatspecs = FormatSpecsListToDict(formatspecs);
1981 delimiter = formatspecs['format_delimiter'];
1982 if(hasattr(infile, "read") or hasattr(infile, "write")):
1983 fp = infile;
1984 fp.seek(0, 0);
1985 fp = UncompressArchiveFile(fp, formatspecs);
1986 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1987 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1988 return False;
1989 if(not fp):
1990 return False;
1991 fp.seek(0, 0);
1992 elif(infile=="-"):
1993 fp = BytesIO();
1994 if(hasattr(sys.stdin, "buffer")):
1995 shutil.copyfileobj(sys.stdin.buffer, fp);
1996 else:
1997 shutil.copyfileobj(sys.stdin, fp);
1998 fp.seek(0, 0);
1999 fp = UncompressArchiveFile(fp, formatspecs);
2000 if(not fp):
2001 return False;
2002 fp.seek(0, 0);
2003 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2004 fp = download_file_from_internet_file(infile);
2005 fp = UncompressArchiveFile(fp, formatspecs);
2006 fp.seek(0, 0);
2007 if(not fp):
2008 return False;
2009 fp.seek(0, 0);
2010 else:
2011 infile = RemoveWindowsPath(infile);
2012 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
2013 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
2014 return False;
2015 compresscheck = CheckCompressionType(infile, formatspecs, True);
2016 if(not compresscheck):
2017 fextname = os.path.splitext(infile)[1];
2018 if(fextname==".gz"):
2019 compresscheck = "gzip";
2020 elif(fextname==".bz2"):
2021 compresscheck = "bzip2";
2022 elif(fextname==".zst"):
2023 compresscheck = "zstd";
2024 elif(fextname==".lz4" or fextname==".clz4"):
2025 compresscheck = "lz4";
2026 elif(fextname==".lzo" or fextname==".lzop"):
2027 compresscheck = "lzo";
2028 elif(fextname==".lzma"):
2029 compresscheck = "lzma";
2030 elif(fextname==".xz"):
2031 compresscheck = "xz";
2032 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
2033 compresscheck = "zlib";
2034 else:
2035 return False;
2036 if(not compresscheck):
2037 return False;
2038 fp = UncompressFile(infile, formatspecs, "rb");
2039 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
2040
2041 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
2042 outdata = str(indata) + delimiter;
2043 return outdata;
2044
2045 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
2046 outdata = "";
2047 inum = 0;
2048 il = len(indata);
2049 while(inum < il):
2050 outdata = outdata + AppendNullByte(indata[inum], delimiter);
2051 inum = inum + 1;
2052 return outdata;
2053
2054 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
2055 formatspecs = FormatSpecsListToDict(formatspecs);
2056 delimiter = formatspecs['format_delimiter'];
2057 catver = formatspecs['format_ver'];
2058 fileheaderver = str(int(catver.replace(".", "")));
2059 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
2060 fp.write(fileheader.encode('UTF-8'));
2061 fnumfiles = format(int(numfiles), 'x').lower();
2062 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
2063 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2064 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
2065 fp.write(fnumfilesa.encode('UTF-8'));
2066 try:
2067 fp.flush();
2068 os.fsync(fp.fileno());
2069 except io.UnsupportedOperation:
2070 pass;
2071 except AttributeError:
2072 pass;
2073 except OSError as e:
2074 pass;
2075 return fp;
2076
2077 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
2078 formatspecs = FormatSpecsListToDict(formatspecs);
2079 AppendFileHeader(fp, 0, checksumtype, formatspecs);
2080 return fp;
2081
2082 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
2083 formatspecs = FormatSpecsListToDict(formatspecs);
2084 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2085 if(os.path.exists(outfile)):
2086 try:
2087 os.unlink(outfile);
2088 except OSError as e:
2089 pass;
2090 if(outfile=="-"):
2091 verbose = False;
2092 catfpfp = BytesIO();
2093 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2094 catfp = outfile;
2095 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2096 catfp = BytesIO();
2097 else:
2098 fbasename = os.path.splitext(outfile)[0];
2099 fextname = os.path.splitext(outfile)[1];
2100 if(not compresswholefile and fextname in outextlistwd):
2101 compresswholefile = True;
2102 catfp = CompressOpenFile(outfile, True, compressionlevel);
2103 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
2104 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2105 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2106 try:
2107 catfp.flush();
2108 os.fsync(catfp.fileno());
2109 except io.UnsupportedOperation:
2110 pass;
2111 except AttributeError:
2112 pass;
2113 except OSError as e:
2114 pass;
2115 if(outfile=="-"):
2116 catfp.seek(0, 0);
2117 if(hasattr(sys.stdout, "buffer")):
2118 shutil.copyfileobj(catfp, sys.stdout.buffer);
2119 else:
2120 shutil.copyfileobj(catfp, sys.stdout);
2121 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2122 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2123 catfp.seek(0, 0);
2124 upload_file_to_internet_file(catfp, outfile);
2125 if(returnfp):
2126 catfp.seek(0, 0);
2127 return catfp;
2128 else:
2129 catfp.close();
2130 return True;
2131
2132 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
2133 formatspecs = FormatSpecsListToDict(formatspecs);
2134 extrafields = format(len(extradata), 'x').lower();
2135 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
2136 if(len(extradata)>0):
2137 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
2138 extrasizelen = format(len(extrasizestr), 'x').lower();
2139 catoutlen = len(filevalues) + len(extradata) + 5;
2140 catoutlenhex = format(catoutlen, 'x').lower();
2141 catoutlist = filevalues;
2142 catoutlist.insert(0, catoutlenhex);
2143 catoutlist.append(extrasizelen);
2144 catoutlist.append(extrafields);
2145 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
2146 if(len(extradata)>0):
2147 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
2148 if(len(filecontent)==0):
2149 checksumlist = [checksumtype, "none"];
2150 else:
2151 checksumlist = [checksumtype, checksumtype];
2152 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
2153 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2154 if(len(filecontent)==0):
2155 catfilecontentcshex = GetFileChecksum(filecontent, "none", False, formatspecs);
2156 else:
2157 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
2158 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
2159 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
2160 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
2161 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2162 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
2163 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2164 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
2165 catfileout = catfileoutstrecd + filecontent + nullstrecd;
2166 fp.write(catfileout);
2167 try:
2168 fp.flush();
2169 os.fsync(fp.fileno());
2170 except io.UnsupportedOperation:
2171 pass;
2172 except AttributeError:
2173 pass;
2174 except OSError as e:
2175 pass;
2176 return fp;
2177
2178 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2179 formatspecs = FormatSpecsListToDict(formatspecs);
2180 advancedlist = formatspecs['use_advanced_list'];
2181 altinode = formatspecs['use_alt_inode'];
2182 if(verbose):
2183 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2184 if(infiles=="-"):
2185 for line in sys.stdin:
2186 infilelist.append(line.strip());
2187 infilelist = list(filter(None, infilelist));
2188 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2189 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2190 return False;
2191 with UncompressFile(infiles, formatspecs, "r") as finfile:
2192 for line in finfile:
2193 infilelist.append(line.strip());
2194 infilelist = list(filter(None, infilelist));
2195 else:
2196 if(isinstance(infiles, (list, tuple, ))):
2197 infilelist = list(filter(None, infiles));
2198 elif(isinstance(infiles, (str, ))):
2199 infilelist = list(filter(None, [infiles]));
2200 if(advancedlist):
2201 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2202 else:
2203 GetDirList = ListDir(infilelist, followlink, False);
2204 if(not GetDirList):
2205 return False;
2206 curinode = 0;
2207 curfid = 0;
2208 inodelist = [];
2209 inodetofile = {};
2210 filetoinode = {};
2211 inodetocatinode = {};
2212 numfiles = int(len(GetDirList));
2213 fnumfiles = format(numfiles, 'x').lower();
2214 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
2215 for curfname in GetDirList:
2216 if(re.findall(r"^[.|/]", curfname)):
2217 fname = curfname;
2218 else:
2219 fname = "./"+curfname;
2220 if(verbose):
2221 VerbosePrintOut(fname);
2222 if(not followlink or followlink is None):
2223 fstatinfo = os.lstat(fname);
2224 else:
2225 fstatinfo = os.stat(fname);
2226 fpremode = fstatinfo.st_mode;
2227 finode = fstatinfo.st_ino;
2228 flinkcount = fstatinfo.st_nlink;
2229 ftype = 0;
2230 if(stat.S_ISREG(fpremode)):
2231 ftype = 0;
2232 elif(stat.S_ISLNK(fpremode)):
2233 ftype = 2;
2234 elif(stat.S_ISCHR(fpremode)):
2235 ftype = 3;
2236 elif(stat.S_ISBLK(fpremode)):
2237 ftype = 4;
2238 elif(stat.S_ISDIR(fpremode)):
2239 ftype = 5;
2240 elif(stat.S_ISFIFO(fpremode)):
2241 ftype = 6;
2242 elif(stat.S_ISSOCK(fpremode)):
2243 ftype = 8;
2244 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2245 ftype = 9;
2246 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2247 ftype = 10;
2248 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2249 ftype = 11;
2250 else:
2251 ftype = 0;
2252 flinkname = "";
2253 fcurfid = format(int(curfid), 'x').lower();
2254 if(not followlink and finode!=0):
2255 if(ftype!=1):
2256 if(finode in inodelist):
2257 ftype = 1;
2258 flinkname = inodetofile[finode];
2259 if(altinode):
2260 fcurinode = format(int(finode), 'x').lower();
2261 else:
2262 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2263 if(finode not in inodelist):
2264 inodelist.append(finode);
2265 inodetofile.update({finode: fname});
2266 inodetocatinode.update({finode: curinode});
2267 if(altinode):
2268 fcurinode = format(int(finode), 'x').lower();
2269 else:
2270 fcurinode = format(int(curinode), 'x').lower();
2271 curinode = curinode + 1;
2272 else:
2273 fcurinode = format(int(curinode), 'x').lower();
2274 curinode = curinode + 1;
2275 curfid = curfid + 1;
2276 if(ftype==2):
2277 flinkname = os.readlink(fname);
2278 fdev = fstatinfo.st_dev;
2279 getfdev = GetDevMajorMinor(fdev);
2280 fdev_minor = getfdev[0];
2281 fdev_major = getfdev[1];
2282 frdev = fstatinfo.st_dev;
2283 if(hasattr(fstatinfo, "st_rdev")):
2284 frdev = fstatinfo.st_rdev;
2285 else:
2286 frdev = fstatinfo.st_dev;
2287 getfrdev = GetDevMajorMinor(frdev);
2288 frdev_minor = getfrdev[0];
2289 frdev_major = getfrdev[1];
2290 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2291 fsize = format(int("0"), 'x').lower();
2292 elif(ftype==0 or ftype==7):
2293 fsize = format(int(fstatinfo.st_size), 'x').lower();
2294 else:
2295 fsize = format(int(fstatinfo.st_size)).lower();
2296 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2297 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2298 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2299 if(hasattr(fstatinfo, "st_birthtime")):
2300 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2301 else:
2302 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2303 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2304 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2305 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2306 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2307 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2308 funame = "";
2309 try:
2310 import pwd;
2311 try:
2312 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2313 funame = userinfo.pw_name;
2314 except KeyError:
2315 funame = "";
2316 except ImportError:
2317 funame = "";
2318 fgname = "";
2319 try:
2320 import grp;
2321 try:
2322 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2323 fgname = groupinfo.gr_name;
2324 except KeyError:
2325 fgname = "";
2326 except ImportError:
2327 fgname = "";
2328 fdev_minor = format(int(fdev_minor), 'x').lower();
2329 fdev_major = format(int(fdev_major), 'x').lower();
2330 frdev_minor = format(int(frdev_minor), 'x').lower();
2331 frdev_major = format(int(frdev_major), 'x').lower();
2332 finode = format(int(finode), 'x').lower();
2333 flinkcount = format(int(flinkcount), 'x').lower();
2334 if(hasattr(fstatinfo, "st_file_attributes")):
2335 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2336 else:
2337 fwinattributes = format(int(0), 'x').lower();
2338 fcompression = "";
2339 fcsize = format(int(0), 'x').lower();
2340 fcontents = BytesIO();
2341 chunk_size = 1024;
2342 if(ftype==0 or ftype==7):
2343 with open(fname, "rb") as fpc:
2344 shutil.copyfileobj(fpc, fcontents);
2345 if(not compresswholefile):
2346 fcontents.seek(0, 2);
2347 ucfsize = fcontents.tell();
2348 fcontents.seek(0, 0);
2349 if(compression=="auto"):
2350 ilsize = len(compressionlistalt);
2351 ilmin = 0;
2352 ilcsize = [];
2353 while(ilmin < ilsize):
2354 cfcontents = BytesIO();
2355 shutil.copyfileobj(fcontents, cfcontents);
2356 fcontents.seek(0, 0);
2357 cfcontents.seek(0, 0);
2358 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2359 if(cfcontents):
2360 cfcontents.seek(0, 2);
2361 ilcsize.append(cfcontents.tell());
2362 cfcontents.close();
2363 else:
2364 try:
2365 ilcsize.append(sys.maxint);
2366 except AttributeError:
2367 ilcsize.append(sys.maxsize);
2368 ilmin = ilmin + 1;
2369 ilcmin = ilcsize.index(min(ilcsize));
2370 compression = compressionlistalt[ilcmin];
2371 fcontents.seek(0, 0);
2372 cfcontents = BytesIO();
2373 shutil.copyfileobj(fcontents, cfcontents);
2374 cfcontents.seek(0, 0);
2375 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2376 cfcontents.seek(0, 2);
2377 cfsize = cfcontents.tell();
2378 if(ucfsize > cfsize):
2379 fcsize = format(int(cfsize), 'x').lower();
2380 fcompression = compression;
2381 fcontents.close();
2382 fcontents = cfcontents;
2383 if(followlink and (ftype==1 or ftype==2)):
2384 flstatinfo = os.stat(flinkname);
2385 with open(flinkname, "rb") as fpc:
2386 shutil.copyfileobj(fpc, fcontents);
2387 if(not compresswholefile):
2388 fcontents.seek(0, 2);
2389 ucfsize = fcontents.tell();
2390 fcontents.seek(0, 0);
2391 if(compression=="auto"):
2392 ilsize = len(compressionlistalt);
2393 ilmin = 0;
2394 ilcsize = [];
2395 while(ilmin < ilsize):
2396 cfcontents = BytesIO();
2397 shutil.copyfileobj(fcontents, cfcontents);
2398 fcontents.seek(0, 0);
2399 cfcontents.seek(0, 0);
2400 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2401 if(cfcontents):
2402 cfcontents.seek(0, 2);
2403 ilcsize.append(cfcontents.tell());
2404 cfcontents.close();
2405 else:
2406 try:
2407 ilcsize.append(sys.maxint);
2408 except AttributeError:
2409 ilcsize.append(sys.maxsize);
2410 ilmin = ilmin + 1;
2411 ilcmin = ilcsize.index(min(ilcsize));
2412 compression = compressionlistalt[ilcmin];
2413 fcontents.seek(0, 0);
2414 cfcontents = BytesIO();
2415 shutil.copyfileobj(fcontents, cfcontents);
2416 cfcontents.seek(0, 0);
2417 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2418 cfcontents.seek(0, 2);
2419 cfsize = cfcontents.tell();
2420 if(ucfsize > cfsize):
2421 fcsize = format(int(cfsize), 'x').lower();
2422 fcompression = compression;
2423 fcontents.close();
2424 fcontents = cfcontents;
2425 if(fcompression=="none"):
2426 fcompression = "";
2427 fcontents.seek(0, 0);
2428 ftypehex = format(ftype, 'x').lower();
2429 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
2430 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2431 if(numfiles>0):
2432 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2433 fp.seek(0, 0);
2434 return fp;
2435
2436 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2437 formatspecs = FormatSpecsListToDict(formatspecs);
2438 if(verbose):
2439 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2440 GetDirList = inlist;
2441 if(not GetDirList):
2442 return False;
2443 curinode = 0;
2444 curfid = 0;
2445 inodelist = [];
2446 inodetofile = {};
2447 filetoinode = {};
2448 inodetocatinode = {};
2449 numfiles = int(len(GetDirList));
2450 fnumfiles = format(numfiles, 'x').lower();
2451 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
2452 for curfname in GetDirList:
2453 ftype = format(curfname[0], 'x').lower();
2454 if(re.findall(r"^[.|/]", curfname[1])):
2455 fname = curfname[1];
2456 else:
2457 fname = "./"+curfname[1];
2458 fbasedir = os.path.dirname(fname);
2459 flinkname = curfname[2];
2460 fsize = format(curfname[3], 'x').lower();
2461 fatime = format(curfname[4], 'x').lower();
2462 fmtime = format(curfname[5], 'x').lower();
2463 fctime = format(curfname[6], 'x').lower();
2464 fbtime = format(curfname[7], 'x').lower();
2465 fmode = format(curfname[8], 'x').lower();
2466 fwinattributes = format(curfname[9], 'x').lower();
2467 fcompression = curfname[10];
2468 fcsize = format(curfname[11], 'x').lower();
2469 fuid = format(curfname[12], 'x').lower();
2470 funame = curfname[13];
2471 fgid = format(curfname[14], 'x').lower();
2472 fgname = curfname[15];
2473 fid = format(curfname[16], 'x').lower();
2474 finode = format(curfname[17], 'x').lower();
2475 flinkcount = format(curfname[18], 'x').lower();
2476 fdev_minor = format(curfname[19], 'x').lower();
2477 fdev_major = format(curfname[20], 'x').lower();
2478 frdev_minor = format(curfname[21], 'x').lower();
2479 frdev_major = format(curfname[22], 'x').lower();
2480 fseeknextfile = curfname[23];
2481 extradata = curfname[24];
2482 fheaderchecksumtype = curfname[25];
2483 fcontentchecksumtype = curfname[26];
2484 fcontents = curfname[27];
2485 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
2486 fcontents.seek(0, 0);
2487 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2488 if(numfiles>0):
2489 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2490 return fp;
2491
2492 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
2493 formatspecs = FormatSpecsListToDict(formatspecs);
2494 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2495 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
2496
2497 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2498 formatspecs = FormatSpecsListToDict(formatspecs);
2499 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2500 if(os.path.exists(outfile)):
2501 try:
2502 os.unlink(outfile);
2503 except OSError as e:
2504 pass;
2505 if(outfile=="-"):
2506 verbose = False;
2507 catfpfp = BytesIO();
2508 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2509 catfp = outfile;
2510 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2511 catfp = BytesIO();
2512 else:
2513 fbasename = os.path.splitext(outfile)[0];
2514 fextname = os.path.splitext(outfile)[1];
2515 if(not compresswholefile and fextname in outextlistwd):
2516 compresswholefile = True;
2517 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2518 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2519 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2520 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2521 try:
2522 catfp.flush();
2523 os.fsync(catfp.fileno());
2524 except io.UnsupportedOperation:
2525 pass;
2526 except AttributeError:
2527 pass;
2528 except OSError as e:
2529 pass;
2530 if(outfile=="-"):
2531 catfp.seek(0, 0);
2532 if(hasattr(sys.stdout, "buffer")):
2533 shutil.copyfileobj(catfp, sys.stdout.buffer);
2534 else:
2535 shutil.copyfileobj(catfp, sys.stdout);
2536 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2537 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2538 catfp.seek(0, 0);
2539 upload_file_to_internet_file(catfp, outfile);
2540 if(returnfp):
2541 catfp.seek(0, 0);
2542 return catfp;
2543 else:
2544 catfp.close();
2545 return True;
2546
2547 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2548 formatspecs = FormatSpecsListToDict(formatspecs);
2549 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2550 if(os.path.exists(outfile)):
2551 try:
2552 os.unlink(outfile);
2553 except OSError as e:
2554 pass;
2555 if(outfile=="-"):
2556 verbose = False;
2557 catfpfp = BytesIO();
2558 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2559 catfp = outfile;
2560 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2561 catfp = BytesIO();
2562 else:
2563 fbasename = os.path.splitext(outfile)[0];
2564 fextname = os.path.splitext(outfile)[1];
2565 if(not compresswholefile and fextname in outextlistwd):
2566 compresswholefile = True;
2567 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2568 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2569 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2570 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2571 try:
2572 catfp.flush();
2573 os.fsync(catfp.fileno());
2574 except io.UnsupportedOperation:
2575 pass;
2576 except AttributeError:
2577 pass;
2578 except OSError as e:
2579 pass;
2580 if(outfile=="-"):
2581 catfp.seek(0, 0);
2582 if(hasattr(sys.stdout, "buffer")):
2583 shutil.copyfileobj(catfp, sys.stdout.buffer);
2584 else:
2585 shutil.copyfileobj(catfp, sys.stdout);
2586 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2587 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2588 catfp.seek(0, 0);
2589 upload_file_to_internet_file(catfp, outfile);
2590 if(returnfp):
2591 catfp.seek(0, 0);
2592 return catfp;
2593 else:
2594 catfp.close();
2595 return True;
2596
2597 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2598 formatspecs = FormatSpecsListToDict(formatspecs);
2599 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2600 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2601
2602 def PrintPermissionString(fchmode, ftype):
2603 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2604 permissionstr = "";
2605 for fmodval in str(oct(fchmode))[-3:]:
2606 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2607 if(ftype==0 or ftype==7):
2608 permissionstr = "-" + permissionstr;
2609 if(ftype==1):
2610 permissionstr = "h" + permissionstr;
2611 if(ftype==2):
2612 permissionstr = "l" + permissionstr;
2613 if(ftype==3):
2614 permissionstr = "c" + permissionstr;
2615 if(ftype==4):
2616 permissionstr = "b" + permissionstr;
2617 if(ftype==5):
2618 permissionstr = "d" + permissionstr;
2619 if(ftype==6):
2620 permissionstr = "f" + permissionstr;
2621 if(ftype==8):
2622 permissionstr = "D" + permissionstr;
2623 if(ftype==9):
2624 permissionstr = "p" + permissionstr;
2625 if(ftype==10):
2626 permissionstr = "w" + permissionstr;
2627 try:
2628 permissionoutstr = stat.filemode(fchmode);
2629 except AttributeError:
2630 permissionoutstr = permissionstr;
2631 except KeyError:
2632 permissionoutstr = permissionstr;
2633 return permissionoutstr;
2634
2635 def PrintPermissionStringAlt(fchmode, ftype):
2636 permissions = {
2637 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2638 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2639 };
2640 # Translate file mode into permission string
2641 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2642 # Append file type indicator
2643 type_indicators = {
2644 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2645 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2646 };
2647 file_type = type_indicators.get(ftype, '-');
2648 permissionstr = file_type + permissionstr;
2649 try:
2650 permissionoutstr = stat.filemode(fchmode);
2651 except AttributeError:
2652 permissionoutstr = permissionstr;
2653 return permissionoutstr;
2654
2655 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2656 formatspecs = FormatSpecsListToDict(formatspecs);
2657 if(hasattr(infile, "read") or hasattr(infile, "write")):
2658 catfp = infile;
2659 else:
2660 try:
2661 catfp = open(infile, "rb");
2662 except FileNotFoundError:
2663 return False;
2664 filetype = False;
2665 catfp.seek(0, 0);
2666 prefp = catfp.read(2);
2667 if(prefp==binascii.unhexlify("1f8b")):
2668 filetype = "gzip";
2669 if(prefp==binascii.unhexlify("7801")):
2670 filetype = "zlib";
2671 if(prefp==binascii.unhexlify("785e")):
2672 filetype = "zlib";
2673 if(prefp==binascii.unhexlify("789c")):
2674 filetype = "zlib";
2675 if(prefp==binascii.unhexlify("78da")):
2676 filetype = "zlib";
2677 catfp.seek(0, 0);
2678 prefp = catfp.read(3);
2679 if(prefp==binascii.unhexlify("425a68")):
2680 filetype = "bzip2";
2681 if(prefp==binascii.unhexlify("5d0000")):
2682 filetype = "lzma";
2683 catfp.seek(0, 0);
2684 prefp = catfp.read(4);
2685 if(prefp==binascii.unhexlify("28b52ffd")):
2686 filetype = "zstd";
2687 if(prefp==binascii.unhexlify("04224d18")):
2688 filetype = "lz4";
2689 if(prefp==binascii.unhexlify("504B0304")):
2690 filetype = "zipfile";
2691 catfp.seek(0, 0);
2692 prefp = catfp.read(5);
2693 if(prefp==binascii.unhexlify("7573746172")):
2694 filetype = "tarfile";
2695 catfp.seek(0, 0);
2696 prefp = catfp.read(6);
2697 if(prefp==binascii.unhexlify("fd377a585a00")):
2698 filetype = "lzma";
2699 if(prefp==binascii.unhexlify("377abcaf271c")):
2700 filetype = "7zipfile";
2701 catfp.seek(0, 0);
2702 prefp = catfp.read(7);
2703 if(prefp==binascii.unhexlify("526172211a0700")):
2704 filetype = "rarfile";
2705 if(prefp==binascii.unhexlify("43617446696c65")):
2706 filetype = "catfile";
2707 catfp.seek(0, 0);
2708 prefp = catfp.read(8);
2709 if(prefp==binascii.unhexlify("526172211a070100")):
2710 filetype = "rarfile";
2711 catfp.seek(0, 0);
2712 prefp = catfp.read(formatspecs['format_len']);
2713 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2714 filetype = formatspecs['format_lower'];
2715 catfp.seek(0, 0);
2716 prefp = catfp.read(9);
2717 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2718 filetype = "lzo";
2719 catfp.seek(0, 0);
2720 prefp = catfp.read(10);
2721 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2722 filetype = "tarfile";
2723 catfp.seek(0, 0);
2724 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4" or filetype=="zlib"):
2725 if(TarFileCheck(catfp)):
2726 filetype = "tarfile";
2727 if(not filetype):
2728 if(TarFileCheck(catfp)):
2729 filetype = "tarfile";
2730 elif(zipfile.is_zipfile(catfp)):
2731 filetype = "zipfile";
2732 elif(rarfile_support and (rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp))):
2733 filetype = "rarile";
2734 elif(py7zr_support and py7zr.is_7zfile(catfp)):
2735 return "7zipfile";
2736 else:
2737 filetype = False;
2738 catfp.seek(0, 0);
2739 if(closefp):
2740 catfp.close();
2741 return filetype;
2742
2743 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2744 formatspecs = FormatSpecsListToDict(formatspecs);
2745 try:
2746 instringsfile = BytesIO(instring);
2747 except TypeError:
2748 instringsfile = BytesIO(instring.encode("UTF-8"));
2749 return CheckCompressionType(instringsfile, formatspecs, closefp);
2750
2751 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2752 formatspecs = FormatSpecsListToDict(formatspecs);
2753 compresscheck = CheckCompressionType(fp, formatspecs, False);
2754 if(compresscheck=="gzip" or compresscheck=="gz"):
2755 return archivefile_gzip_mimetype;
2756 if(compresscheck=="zlib" or (compresscheck=="zz" or compresscheck=="zl" or compresscheck=="zlib")):
2757 return archivefile_zlib_mimetype;
2758 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2759 return archivefile_bzip2_mimetype;
2760 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2761 return archivefile_zstandard_mimetype;
2762 if(compresscheck=="lz4"):
2763 return archivefile_lz4_mimetype;
2764 if(compresscheck=="lzo" or compresscheck=="lzop"):
2765 return archivefile_lzop_mimetype;
2766 if(compresscheck=="lzma"):
2767 return archivefile_lzma_mimetype;
2768 if(compresscheck=="xz"):
2769 return archivefile_xz_mimetype;
2770 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2771 return archivefile_cat_mimetype;
2772 if(not compresscheck):
2773 return False;
2774 return False;
2775
2776 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2777 formatspecs = FormatSpecsListToDict(formatspecs);
2778 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2779 return False;
2780 compresscheck = CheckCompressionType(fp, formatspecs, False);
2781 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2782 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2783 elif(compresscheck=="bzip2" and compresscheck in compressionsupport):
2784 catfp = bz2.BZ2File(fp);
2785 elif(compresscheck=="zstd" and compresscheck in compressionsupport):
2786 catfp = zstd.ZstdDecompressor().stream_reader(fp);
2787 elif(compresscheck=="lz4" and compresscheck in compressionsupport):
2788 catfp = lz4.frame.open_fp(fp, mode='rb');
2789 elif((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2790 catfp = BytesIO();
2791 catfp.write(lzo.decompress(fp.read()));
2792 elif((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2793 catfp = lzma.LZMAFile(fp);
2794 elif(compresscheck=="zlib" and compresscheck in compressionsupport):
2795 catfp = ZlibFile(fileobj=fp, mode="rb");
2796 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2797 catfp = fp;
2798 if(not compresscheck):
2799 try:
2800 import lzma;
2801 except ImportError:
2802 try:
2803 from backports import lzma
2804 except ImportError:
2805 return False;
2806 catfp = BytesIO();
2807 with fp as fpcontent:
2808 try:
2809 catfp.write(lzma.decompress(fp.read()));
2810 except lzma.LZMAError:
2811 return False;
2812 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2813 fp.close();
2814 return catfp;
2815
2816 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2817
2818 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2819 formatspecs = FormatSpecsListToDict(formatspecs);
2820 compresscheck = CheckCompressionType(infile, formatspecs, False);
2821 if(sys.version_info[0]==2 and compresscheck):
2822 if(mode=="rt"):
2823 mode = "r";
2824 if(mode=="wt"):
2825 mode = "w";
2826 try:
2827 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2828 try:
2829 filefp = gzip.open(infile, mode, encoding="UTF-8");
2830 except (ValueError, TypeError) as e:
2831 filefp = gzip.open(infile, mode);
2832 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2833 try:
2834 filefp = bz2.open(infile, mode, encoding="UTF-8");
2835 except (ValueError, TypeError) as e:
2836 filefp = bz2.open(infile, mode);
2837 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2838 try:
2839 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2840 except (ValueError, TypeError) as e:
2841 filefp = zstandard.open(infile, mode);
2842 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2843 try:
2844 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2845 except (ValueError, TypeError) as e:
2846 filefp = lz4.frame.open(infile, mode);
2847 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2848 try:
2849 filefp = lzo.open(infile, mode, encoding="UTF-8");
2850 except (ValueError, TypeError) as e:
2851 filefp = lzo.open(infile, mode);
2852 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2853 try:
2854 filefp = lzma.open(infile, mode, encoding="UTF-8");
2855 except (ValueError, TypeError) as e:
2856 filefp = lzma.open(infile, mode);
2857 if(compresscheck=="zlib" and compresscheck in compressionsupport):
2858 filefp = ZlibFile(infile, mode=mode);
2859 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2860 try:
2861 filefp = open(infile, mode, encoding="UTF-8");
2862 except (ValueError, TypeError) as e:
2863 filefp = open(infile, mode);
2864 if(not compresscheck):
2865 try:
2866 filefp = open(infile, mode, encoding="UTF-8");
2867 except (ValueError, TypeError) as e:
2868 filefp = open(infile, mode);
2869 except FileNotFoundError:
2870 return False;
2871 try:
2872 filefp.write_through = True;
2873 except AttributeError:
2874 pass;
2875 return filefp;
2876
2877 def UncompressString(infile):
2878 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2879 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2880 fileuz = gzip.decompress(infile);
2881 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2882 fileuz = bz2.decompress(infile);
2883 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2884 try:
2885 import zstandard;
2886 except ImportError:
2887 return False;
2888 fileuz = zstandard.decompress(infile);
2889 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2890 fileuz = lz4.frame.decompress(infile);
2891 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2892 fileuz = lzo.decompress(infile);
2893 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2894 fileuz = lzma.decompress(infile);
2895 if(compresscheck=="zlib" and compresscheck in compressionsupport):
2896 fileuz = zlib.decompress(infile);
2897 if(not compresscheck):
2898 fileuz = infile;
2899 if(hasattr(fileuz, 'decode')):
2900 fileuz = fileuz.decode("UTF-8");
2901 return fileuz;
2902
2903 def UncompressStringAlt(infile):
2904 filefp = StringIO();
2905 outstring = UncompressString(infile);
2906 filefp.write(outstring);
2907 filefp.seek(0, 0);
2908 return filefp;
2909
2910 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2911 formatspecs = FormatSpecsListToDict(formatspecs);
2912 compresscheck = CheckCompressionType(infile, formatspecs, False);
2913 if(not compresscheck):
2914 fextname = os.path.splitext(infile)[1];
2915 if(fextname==".gz"):
2916 compresscheck = "gzip";
2917 elif(fextname==".bz2"):
2918 compresscheck = "bzip2";
2919 elif(fextname==".zst"):
2920 compresscheck = "zstd";
2921 elif(fextname==".lz4"):
2922 compresscheck = "lz4";
2923 elif(fextname==".lzo" or fextname==".lzop"):
2924 compresscheck = "lzo";
2925 elif(fextname==".lzma"):
2926 compresscheck = "lzma";
2927 elif(fextname==".xz"):
2928 compresscheck = "xz";
2929 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
2930 compresscheck = "zlib";
2931 else:
2932 return False;
2933 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4" or compresscheck=="zlib"):
2934 if(TarFileCheck(infile)):
2935 filetype = "tarfile";
2936 if(not compresscheck):
2937 if(TarFileCheck(infile)):
2938 return "tarfile";
2939 elif(zipfile.is_zipfile(infile)):
2940 return "zipfile";
2941 elif(rarfile_support and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
2942 return "rarile";
2943 elif(py7zr_support and py7zr.is_7zfile(infile)):
2944 return "7zipfile";
2945 else:
2946 return False;
2947 return False;
2948 if(compresscheck=="catfile"):
2949 return "catfile";
2950 if(compresscheck==formatspecs['format_lower']):
2951 return formatspecs['format_lower'];
2952 if(compresscheck=="tarfile"):
2953 return "tarfile";
2954 if(compresscheck=="zipfile"):
2955 return "zipfile";
2956 if(rarfile_support and compresscheck=="rarfile"):
2957 return "rarfile";
2958 if(py7zr_support and compresscheck=="7zipfile" and py7zr.is_7zfile(infile)):
2959 return "7zipfile";
2960 if(hasattr(infile, "read") or hasattr(infile, "write")):
2961 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2962 else:
2963 try:
2964 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2965 catfp = gzip.GzipFile(infile, "rb");
2966 elif(compresscheck=="bzip2" and compresscheck in compressionsupport):
2967 catfp = bz2.BZ2File(infile, "rb");
2968 elif(compresscheck=="lz4" and compresscheck in compressionsupport):
2969 catfp = lz4.frame.open(infile, "rb");
2970 elif(compresscheck=="zstd" and compresscheck in compressionsupport):
2971 catfp = zstandard.open(infile, "rb");
2972 elif((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2973 catfp = lzo.open(infile, "rb");
2974 elif((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2975 catfp = lzma.open(infile, "rb");
2976 elif(compresscheck=="zlib" and compresscheck in compressionsupport):
2977 catfp = ZlibFile(infile, mode="rb");
2978 else:
2979 catfp = open(infile, "rb");
2980 except FileNotFoundError:
2981 return False;
2982 filetype = False;
2983 prefp = catfp.read(5);
2984 if(prefp==binascii.unhexlify("7573746172")):
2985 filetype = "tarfile";
2986 catfp.seek(0, 0);
2987 prefp = catfp.read(7);
2988 if(prefp==binascii.unhexlify("43617446696c65")):
2989 filetype = "catfile";
2990 catfp.seek(0, 0);
2991 prefp = catfp.read(formatspecs['format_len']);
2992 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2993 filetype = formatspecs['format_lower'];
2994 catfp.seek(0, 0);
2995 prefp = catfp.read(10);
2996 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2997 filetype = "tarfile";
2998 catfp.seek(0, 0);
2999 if(closefp):
3000 catfp.close();
3001 return filetype;
3002
3003 def GZipCompress(data, compresslevel=9):
3004 if("gzip" not in compressionsupport):
3005 return False;
3006 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
3007 tmpfp.close();
3008 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
3009 tmpfp.write(data);
3010 tmpfp.close();
3011 try:
3012 catfp = open(tmpfp.name, "rb");
3013 except FileNotFoundError:
3014 return False;
3015 catdata = catfp.read();
3016 catfp.close();
3017 return catdata;
3018
3019 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
3020 formatspecs = FormatSpecsListToDict(formatspecs);
3021 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
3022 return False;
3023 fp.seek(0, 0);
3024 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3025 compression = "auto";
3026 if(compression not in compressionlist and compression is None):
3027 compression = "auto";
3028 if(compression=="gzip" and compression in compressionsupport):
3029 catfp = BytesIO();
3030 if(compressionlevel is None):
3031 compressionlevel = 9;
3032 else:
3033 compressionlevel = int(compressionlevel);
3034 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
3035 if(compression=="bzip2" and compression in compressionsupport):
3036 catfp = BytesIO();
3037 if(compressionlevel is None):
3038 compressionlevel = 9;
3039 else:
3040 compressionlevel = int(compressionlevel);
3041 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
3042 if(compression=="lz4" and compression in compressionsupport):
3043 catfp = BytesIO();
3044 if(compressionlevel is None):
3045 compressionlevel = 9;
3046 else:
3047 compressionlevel = int(compressionlevel);
3048 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
3049 if((compression=="lzo" or compression=="lzop") and compression in compressionsupport):
3050 catfp = BytesIO();
3051 if(compressionlevel is None):
3052 compressionlevel = 9;
3053 else:
3054 compressionlevel = int(compressionlevel);
3055 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
3056 if(compression=="zstd" and compression in compressionsupport):
3057 catfp = BytesIO();
3058 if(compressionlevel is None):
3059 compressionlevel = 10;
3060 else:
3061 compressionlevel = int(compressionlevel);
3062 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
3063 if(compression=="lzma" and compression in compressionsupport):
3064 catfp = BytesIO();
3065 if(compressionlevel is None):
3066 compressionlevel = 9;
3067 else:
3068 compressionlevel = int(compressionlevel);
3069 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
3070 if(compression=="xz" and compression in compressionsupport):
3071 catfp = BytesIO();
3072 if(compressionlevel is None):
3073 compressionlevel = 9;
3074 else:
3075 compressionlevel = int(compressionlevel);
3076 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
3077 if(compression=="zlib" and compression in compressionsupport):
3078 catfp = BytesIO();
3079 if(compressionlevel is None):
3080 compressionlevel = 9;
3081 else:
3082 compressionlevel = int(compressionlevel);
3083 catfp.write(zlib.compress(fp.read(), level=compressionlevel));
3084 if(compression=="auto" or compression is None):
3085 catfp = fp;
3086 catfp.seek(0, 0);
3087 return catfp;
3088
3089 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
3090
3091 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
3092 if(outfile is None):
3093 return False;
3094 fbasename = os.path.splitext(outfile)[0];
3095 fextname = os.path.splitext(outfile)[1];
3096 if(compressionlevel is None and fextname!=".zst"):
3097 compressionlevel = 9;
3098 elif(compressionlevel is None and fextname==".zst"):
3099 compressionlevel = 10;
3100 else:
3101 compressionlevel = int(compressionlevel);
3102 if(sys.version_info[0]==2):
3103 mode = "w";
3104 else:
3105 mode = "wb";
3106 try:
3107 if(fextname not in outextlistwd or not compressionenable):
3108 try:
3109 outfp = open(outfile, "wb", encoding="UTF-8");
3110 except (ValueError, TypeError) as e:
3111 outfp = open(outfile, "wb");
3112 elif(fextname==".gz" and "gzip" in compressionsupport):
3113 try:
3114 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
3115 except (ValueError, TypeError) as e:
3116 outfp = gzip.open(outfile, mode, compressionlevel);
3117 elif(fextname==".bz2" and "bzip2" in compressionsupport):
3118 try:
3119 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
3120 except (ValueError, TypeError) as e:
3121 outfp = bz2.open(outfile, mode, compressionlevel);
3122 elif(fextname==".zst" and "zstandard" in compressionsupport):
3123 try:
3124 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
3125 except (ValueError, TypeError) as e:
3126 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
3127 elif(fextname==".xz" and "xz" in compressionsupport):
3128 try:
3129 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
3130 except (ValueError, TypeError) as e:
3131 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
3132 elif(fextname==".lz4" and "lz4" in compressionsupport):
3133 try:
3134 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
3135 except (ValueError, TypeError) as e:
3136 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
3137 elif(fextname==".lzo" and "lzop" in compressionsupport):
3138 try:
3139 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
3140 except (ValueError, TypeError) as e:
3141 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
3142 elif(fextname==".lzma" and "lzma" in compressionsupport):
3143 try:
3144 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
3145 except (ValueError, TypeError) as e:
3146 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
3147 elif((fextname==".zz" or fextname==".zl" or fextname==".zlib") and "zlib" in compressionsupport):
3148 outfp = ZlibFile(outfile, mode=mode, level=compressionlevel);
3149 except FileNotFoundError:
3150 return False;
3151 try:
3152 outfp.write_through = True;
3153 except AttributeError:
3154 pass;
3155 return outfp;
3156
3157 def GetDevMajorMinor(fdev):
3158 retdev = [];
3159 if(hasattr(os, "minor")):
3160 retdev.append(os.minor(fdev));
3161 else:
3162 retdev.append(0);
3163 if(hasattr(os, "major")):
3164 retdev.append(os.major(fdev));
3165 else:
3166 retdev.append(0);
3167 return retdev;
3168
3169 def CheckSumSupport(checkfor, guaranteed=True):
3170 if(guaranteed):
3171 try:
3172 hash_list = sorted(list(hashlib.algorithms_guaranteed));
3173 except AttributeError:
3174 hash_list = sorted(list(hashlib.algorithms));
3175 else:
3176 try:
3177 hash_list = sorted(list(hashlib.algorithms_available));
3178 except AttributeError:
3179 hash_list = sorted(list(hashlib.algorithms));
3180 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
3181 if(checkfor in checklistout):
3182 return True;
3183 else:
3184 return False;
3185
3186 def CheckSumSupportAlt(checkfor, guaranteed=True):
3187 if(guaranteed):
3188 try:
3189 hash_list = sorted(list(hashlib.algorithms_guaranteed));
3190 except AttributeError:
3191 hash_list = sorted(list(hashlib.algorithms));
3192 else:
3193 try:
3194 hash_list = sorted(list(hashlib.algorithms_available));
3195 except AttributeError:
3196 hash_list = sorted(list(hashlib.algorithms));
3197 checklistout = hash_list;
3198 if(checkfor in checklistout):
3199 return True;
3200 else:
3201 return False;
3202
3203 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3204 formatspecs = FormatSpecsListToDict(formatspecs);
3205 advancedlist = formatspecs['use_advanced_list'];
3206 altinode = formatspecs['use_alt_inode'];
3207 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3208 outfile = RemoveWindowsPath(outfile);
3209 checksumtype = checksumtype.lower();
3210 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3211 checksumtype="crc32";
3212 if(checksumtype=="none"):
3213 checksumtype = "";
3214 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3215 compression = "auto";
3216 if(compression not in compressionlist and compression is None):
3217 compression = "auto";
3218 if(verbose):
3219 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3220 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3221 if(os.path.exists(outfile)):
3222 try:
3223 os.unlink(outfile);
3224 except OSError as e:
3225 pass;
3226 if(outfile=="-"):
3227 verbose = False;
3228 catfp = BytesIO();
3229 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3230 catfp = outfile;
3231 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3232 catfp = BytesIO();
3233 else:
3234 fbasename = os.path.splitext(outfile)[0];
3235 fextname = os.path.splitext(outfile)[1];
3236 if(not compresswholefile and fextname in outextlistwd):
3237 compresswholefile = True;
3238 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3239 catver = formatspecs['format_ver'];
3240 fileheaderver = str(int(catver.replace(".", "")));
3241 infilelist = [];
3242 if(infiles=="-"):
3243 for line in sys.stdin:
3244 infilelist.append(line.strip());
3245 infilelist = list(filter(None, infilelist));
3246 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
3247 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
3248 return False;
3249 with UncompressFile(infiles, formatspecs, "r") as finfile:
3250 for line in finfile:
3251 infilelist.append(line.strip());
3252 infilelist = list(filter(None, infilelist));
3253 else:
3254 if(isinstance(infiles, (list, tuple, ))):
3255 infilelist = list(filter(None, infiles));
3256 elif(isinstance(infiles, (str, ))):
3257 infilelist = list(filter(None, [infiles]));
3258 if(advancedlist):
3259 GetDirList = ListDirAdvanced(infilelist, followlink, False);
3260 else:
3261 GetDirList = ListDir(infilelist, followlink, False);
3262 if(not GetDirList):
3263 return False;
3264 curinode = 0;
3265 curfid = 0;
3266 inodelist = [];
3267 inodetofile = {};
3268 filetoinode = {};
3269 inodetocatinode = {};
3270 numfiles = int(len(GetDirList));
3271 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3272 for curfname in GetDirList:
3273 if(re.findall(r"^[.|/]", curfname)):
3274 fname = curfname;
3275 else:
3276 fname = "./"+curfname;
3277 if(verbose):
3278 VerbosePrintOut(fname);
3279 if(not followlink or followlink is None):
3280 fstatinfo = os.lstat(fname);
3281 else:
3282 fstatinfo = os.stat(fname);
3283 fpremode = fstatinfo.st_mode;
3284 finode = fstatinfo.st_ino;
3285 flinkcount = fstatinfo.st_nlink;
3286 ftype = 0;
3287 if(stat.S_ISREG(fpremode)):
3288 ftype = 0;
3289 elif(stat.S_ISLNK(fpremode)):
3290 ftype = 2;
3291 elif(stat.S_ISCHR(fpremode)):
3292 ftype = 3;
3293 elif(stat.S_ISBLK(fpremode)):
3294 ftype = 4;
3295 elif(stat.S_ISDIR(fpremode)):
3296 ftype = 5;
3297 elif(stat.S_ISFIFO(fpremode)):
3298 ftype = 6;
3299 elif(stat.S_ISSOCK(fpremode)):
3300 ftype = 8;
3301 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
3302 ftype = 9;
3303 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
3304 ftype = 10;
3305 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
3306 ftype = 11;
3307 else:
3308 ftype = 0;
3309 flinkname = "";
3310 fcurfid = format(int(curfid), 'x').lower();
3311 if(not followlink and finode!=0):
3312 if(ftype!=1):
3313 if(finode in inodelist):
3314 ftype = 1;
3315 flinkname = inodetofile[finode];
3316 if(altinode):
3317 fcurinode = format(int(finode), 'x').lower();
3318 else:
3319 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
3320 if(finode not in inodelist):
3321 inodelist.append(finode);
3322 inodetofile.update({finode: fname});
3323 inodetocatinode.update({finode: curinode});
3324 if(altinode):
3325 fcurinode = format(int(finode), 'x').lower();
3326 else:
3327 fcurinode = format(int(curinode), 'x').lower();
3328 curinode = curinode + 1;
3329 else:
3330 fcurinode = format(int(curinode), 'x').lower();
3331 curinode = curinode + 1;
3332 curfid = curfid + 1;
3333 if(ftype==2):
3334 flinkname = os.readlink(fname);
3335 fdev = fstatinfo.st_dev;
3336 getfdev = GetDevMajorMinor(fdev);
3337 fdev_minor = getfdev[0];
3338 fdev_major = getfdev[1];
3339 frdev = fstatinfo.st_dev;
3340 if(hasattr(fstatinfo, "st_rdev")):
3341 frdev = fstatinfo.st_rdev;
3342 else:
3343 frdev = fstatinfo.st_dev;
3344 getfrdev = GetDevMajorMinor(frdev);
3345 frdev_minor = getfrdev[0];
3346 frdev_major = getfrdev[1];
3347 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3348 fsize = format(int("0"), 'x').lower();
3349 elif(ftype==0 or ftype==7):
3350 fsize = format(int(fstatinfo.st_size), 'x').lower();
3351 else:
3352 fsize = format(int(fstatinfo.st_size)).lower();
3353 fatime = format(int(fstatinfo.st_atime), 'x').lower();
3354 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
3355 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
3356 if(hasattr(fstatinfo, "st_birthtime")):
3357 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
3358 else:
3359 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
3360 fmode = format(int(fstatinfo.st_mode), 'x').lower();
3361 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
3362 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
3363 fuid = format(int(fstatinfo.st_uid), 'x').lower();
3364 fgid = format(int(fstatinfo.st_gid), 'x').lower();
3365 funame = "";
3366 try:
3367 import pwd;
3368 try:
3369 userinfo = pwd.getpwuid(fstatinfo.st_uid);
3370 funame = userinfo.pw_name;
3371 except KeyError:
3372 funame = "";
3373 except ImportError:
3374 funame = "";
3375 fgname = "";
3376 try:
3377 import grp;
3378 try:
3379 groupinfo = grp.getgrgid(fstatinfo.st_gid);
3380 fgname = groupinfo.gr_name;
3381 except KeyError:
3382 fgname = "";
3383 except ImportError:
3384 fgname = "";
3385 fdev_minor = format(int(fdev_minor), 'x').lower();
3386 fdev_major = format(int(fdev_major), 'x').lower();
3387 frdev_minor = format(int(frdev_minor), 'x').lower();
3388 frdev_major = format(int(frdev_major), 'x').lower();
3389 finode = format(int(finode), 'x').lower();
3390 flinkcount = format(int(flinkcount), 'x').lower();
3391 if(hasattr(fstatinfo, "st_file_attributes")):
3392 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
3393 else:
3394 fwinattributes = format(int(0), 'x').lower();
3395 fcompression = "";
3396 fcsize = format(int(0), 'x').lower();
3397 fcontents = BytesIO();
3398 if(ftype==0 or ftype==7):
3399 with open(fname, "rb") as fpc:
3400 shutil.copyfileobj(fpc, fcontents);
3401 if(not compresswholefile):
3402 fcontents.seek(0, 2);
3403 ucfsize = fcontents.tell();
3404 fcontents.seek(0, 0);
3405 if(compression=="auto"):
3406 ilsize = len(compressionlistalt);
3407 ilmin = 0;
3408 ilcsize = [];
3409 while(ilmin < ilsize):
3410 cfcontents = BytesIO();
3411 shutil.copyfileobj(fcontents, cfcontents);
3412 fcontents.seek(0, 0);
3413 cfcontents.seek(0, 0);
3414 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3415 if(cfcontents):
3416 cfcontents.seek(0, 2);
3417 ilcsize.append(cfcontents.tell());
3418 cfcontents.close();
3419 else:
3420 try:
3421 ilcsize.append(sys.maxint);
3422 except AttributeError:
3423 ilcsize.append(sys.maxsize);
3424 ilmin = ilmin + 1;
3425 ilcmin = ilcsize.index(min(ilcsize));
3426 compression = compressionlistalt[ilcmin];
3427 fcontents.seek(0, 0);
3428 cfcontents = BytesIO();
3429 shutil.copyfileobj(fcontents, cfcontents);
3430 cfcontents.seek(0, 0);
3431 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3432 cfcontents.seek(0, 2);
3433 cfsize = cfcontents.tell();
3434 if(ucfsize > cfsize):
3435 fcsize = format(int(cfsize), 'x').lower();
3436 fcompression = compression;
3437 fcontents.close();
3438 fcontents = cfcontents;
3439 if(fcompression=="none"):
3440 fcompression = "";
3441 if(followlink and (ftype==1 or ftype==2)):
3442 flstatinfo = os.stat(flinkname);
3443 with open(flinkname, "rb") as fpc:
3444 shutil.copyfileobj(fpc, fcontents);
3445 if(not compresswholefile):
3446 fcontents.seek(0, 2);
3447 ucfsize = fcontents.tell();
3448 fcontents.seek(0, 0);
3449 if(compression=="auto"):
3450 ilsize = len(compressionlistalt);
3451 ilmin = 0;
3452 ilcsize = [];
3453 while(ilmin < ilsize):
3454 cfcontents = BytesIO();
3455 shutil.copyfileobj(fcontents, cfcontents);
3456 fcontents.seek(0, 0);
3457 cfcontents.seek(0, 0);
3458 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3459 if(cfcontents):
3460 cfcontents.seek(0, 2);
3461 ilcsize.append(cfcontents.tell());
3462 cfcontents.close();
3463 else:
3464 try:
3465 ilcsize.append(sys.maxint);
3466 except AttributeError:
3467 ilcsize.append(sys.maxsize);
3468 ilmin = ilmin + 1;
3469 ilcmin = ilcsize.index(min(ilcsize));
3470 compression = compressionlistalt[ilcmin];
3471 fcontents.seek(0, 0);
3472 cfcontents = BytesIO();
3473 shutil.copyfileobj(fcontents, cfcontents);
3474 cfcontents.seek(0, 0);
3475 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3476 cfcontents.seek(0, 2);
3477 cfsize = cfcontents.tell();
3478 if(ucfsize > cfsize):
3479 fcsize = format(int(cfsize), 'x').lower();
3480 fcompression = compression;
3481 fcontents.close();
3482 fcontents = cfcontents;
3483 fcontents.seek(0, 0);
3484 ftypehex = format(ftype, 'x').lower();
3485 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3486 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3487 fcontents.close();
3488 if(numfiles>0):
3489 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3490 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3491 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3492 try:
3493 catfp.flush();
3494 os.fsync(catfp.fileno());
3495 except io.UnsupportedOperation:
3496 pass;
3497 except AttributeError:
3498 pass;
3499 except OSError as e:
3500 pass;
3501 if(outfile=="-"):
3502 catfp.seek(0, 0);
3503 if(hasattr(sys.stdout, "buffer")):
3504 shutil.copyfileobj(catfp, sys.stdout.buffer);
3505 else:
3506 shutil.copyfileobj(catfp, sys.stdout);
3507 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3508 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3509 catfp.seek(0, 0);
3510 upload_file_to_internet_file(catfp, outfile);
3511 if(returnfp):
3512 catfp.seek(0, 0);
3513 return catfp;
3514 else:
3515 catfp.close();
3516 return True;
3517
3518 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3519
3520 if(hasattr(shutil, "register_archive_format")):
3521 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3522 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3523 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3524
3525 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3526 formatspecs = FormatSpecsListToDict(formatspecs);
3527 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3528
3529 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3530
3531 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3532 formatspecs = FormatSpecsListToDict(formatspecs);
3533 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3534 outfile = RemoveWindowsPath(outfile);
3535 checksumtype = checksumtype.lower();
3536 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3537 checksumtype="crc32";
3538 if(checksumtype=="none"):
3539 checksumtype = "";
3540 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3541 compression = "auto";
3542 if(compression not in compressionlist and compression is None):
3543 compression = "auto";
3544 if(verbose):
3545 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3546 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3547 if(os.path.exists(outfile)):
3548 try:
3549 os.unlink(outfile);
3550 except OSError as e:
3551 pass;
3552 if(outfile=="-"):
3553 verbose = False;
3554 catfp = BytesIO();
3555 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3556 catfp = outfile;
3557 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3558 catfp = BytesIO();
3559 else:
3560 fbasename = os.path.splitext(outfile)[0];
3561 fextname = os.path.splitext(outfile)[1];
3562 if(not compresswholefile and fextname in outextlistwd):
3563 compresswholefile = True;
3564 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3565 catver = formatspecs['format_ver'];
3566 fileheaderver = str(int(catver.replace(".", "")));
3567 curinode = 0;
3568 curfid = 0;
3569 inodelist = [];
3570 inodetofile = {};
3571 filetoinode = {};
3572 inodetocatinode = {};
3573 if(infile=="-"):
3574 infile = BytesIO();
3575 if(hasattr(sys.stdin, "buffer")):
3576 shutil.copyfileobj(sys.stdin.buffer, infile);
3577 else:
3578 shutil.copyfileobj(sys.stdin, infile);
3579 infile.seek(0, 0);
3580 if(not infile):
3581 return False;
3582 infile.seek(0, 0);
3583 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3584 infile = download_file_from_internet_file(infile);
3585 infile.seek(0, 0);
3586 if(not infile):
3587 return False;
3588 infile.seek(0, 0);
3589 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3590 return False;
3591 elif(os.path.exists(infile) and os.path.isfile(infile)):
3592 try:
3593 if(not tarfile.TarFileCheck(infile)):
3594 return False;
3595 except AttributeError:
3596 if(not TarFileCheck(infile)):
3597 return False;
3598 try:
3599 if(hasattr(infile, "read") or hasattr(infile, "write")):
3600 tarfp = tarfile.open(fileobj=infile, mode="r");
3601 else:
3602 tarfp = tarfile.open(infile, "r");
3603 except FileNotFoundError:
3604 return False;
3605 numfiles = int(len(tarfp.getmembers()));
3606 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3607 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3608 if(re.findall(r"^[.|/]", member.name)):
3609 fname = member.name;
3610 else:
3611 fname = "./"+member.name;
3612 if(verbose):
3613 VerbosePrintOut(fname);
3614 fpremode = member.mode;
3615 ffullmode = member.mode;
3616 flinkcount = 0;
3617 ftype = 0;
3618 if(member.isreg()):
3619 ffullmode = member.mode + stat.S_IFREG;
3620 ftype = 0;
3621 elif(member.isdev()):
3622 ffullmode = member.mode;
3623 ftype = 7;
3624 elif(member.islnk()):
3625 ffullmode = member.mode + stat.S_IFREG;
3626 ftype = 1;
3627 elif(member.issym()):
3628 ffullmode = member.mode + stat.S_IFLNK;
3629 ftype = 2;
3630 elif(member.ischr()):
3631 ffullmode = member.mode + stat.S_IFCHR;
3632 ftype = 3;
3633 elif(member.isblk()):
3634 ffullmode = member.mode + stat.S_IFBLK;
3635 ftype = 4;
3636 elif(member.isdir()):
3637 ffullmode = member.mode + stat.S_IFDIR;
3638 ftype = 5;
3639 elif(member.isfifo()):
3640 ffullmode = member.mode + stat.S_IFIFO;
3641 ftype = 6;
3642 elif(member.issparse()):
3643 ffullmode = member.mode;
3644 ftype = 12;
3645 else:
3646 ffullmode = member.mode;
3647 ftype = 0;
3648 flinkname = "";
3649 fcurfid = format(int(curfid), 'x').lower();
3650 fcurinode = format(int(curfid), 'x').lower();
3651 curfid = curfid + 1;
3652 if(ftype==2):
3653 flinkname = member.linkname;
3654 fdev_minor = format(int(member.devminor), 'x').lower();
3655 fdev_major = format(int(member.devmajor), 'x').lower();
3656 frdev_minor = format(int(member.devminor), 'x').lower();
3657 frdev_major = format(int(member.devmajor), 'x').lower();
3658 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3659 fsize = format(int("0"), 'x').lower();
3660 elif(ftype==0 or ftype==7):
3661 fsize = format(int(member.size), 'x').lower();
3662 else:
3663 fsize = format(int(member.size), 'x').lower();
3664 fatime = format(int(member.mtime), 'x').lower();
3665 fmtime = format(int(member.mtime), 'x').lower();
3666 fctime = format(int(member.mtime), 'x').lower();
3667 fbtime = format(int(member.mtime), 'x').lower();
3668 fmode = format(int(ffullmode), 'x').lower();
3669 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3670 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3671 fuid = format(int(member.uid), 'x').lower();
3672 fgid = format(int(member.gid), 'x').lower();
3673 funame = member.uname;
3674 fgname = member.gname;
3675 flinkcount = format(int(flinkcount), 'x').lower();
3676 fwinattributes = format(int(0), 'x').lower();
3677 fcompression = "";
3678 fcsize = format(int(0), 'x').lower();
3679 fcontents = BytesIO();
3680 if(ftype==0 or ftype==7):
3681 with tarfp.extractfile(member) as fpc:
3682 shutil.copyfileobj(fpc, fcontents);
3683 if(not compresswholefile):
3684 fcontents.seek(0, 2);
3685 ucfsize = fcontents.tell();
3686 fcontents.seek(0, 0);
3687 if(compression=="auto"):
3688 ilsize = len(compressionlistalt);
3689 ilmin = 0;
3690 ilcsize = [];
3691 while(ilmin < ilsize):
3692 cfcontents = BytesIO();
3693 shutil.copyfileobj(fcontents, cfcontents);
3694 fcontents.seek(0, 0);
3695 cfcontents.seek(0, 0);
3696 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3697 if(cfcontents):
3698 cfcontents.seek(0, 2);
3699 ilcsize.append(cfcontents.tell());
3700 cfcontents.close();
3701 else:
3702 try:
3703 ilcsize.append(sys.maxint);
3704 except AttributeError:
3705 ilcsize.append(sys.maxsize);
3706 ilmin = ilmin + 1;
3707 ilcmin = ilcsize.index(min(ilcsize));
3708 compression = compressionlistalt[ilcmin];
3709 fcontents.seek(0, 0);
3710 cfcontents = BytesIO();
3711 shutil.copyfileobj(fcontents, cfcontents);
3712 cfcontents.seek(0, 0);
3713 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3714 cfcontents.seek(0, 2);
3715 cfsize = cfcontents.tell();
3716 if(ucfsize > cfsize):
3717 fcsize = format(int(cfsize), 'x').lower();
3718 fcompression = compression;
3719 fcontents.close();
3720 fcontents = cfcontents;
3721 if(fcompression=="none"):
3722 fcompression = "";
3723 fcontents.seek(0, 0);
3724 ftypehex = format(ftype, 'x').lower();
3725 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3726 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3727 fcontents.close();
3728 if(numfiles>0):
3729 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3730 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3731 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3732 try:
3733 catfp.flush();
3734 os.fsync(catfp.fileno());
3735 except io.UnsupportedOperation:
3736 pass;
3737 except AttributeError:
3738 pass;
3739 except OSError as e:
3740 pass;
3741 if(outfile=="-"):
3742 catfp.seek(0, 0);
3743 if(hasattr(sys.stdout, "buffer")):
3744 shutil.copyfileobj(catfp, sys.stdout.buffer);
3745 else:
3746 shutil.copyfileobj(catfp, sys.stdout);
3747 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3748 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3749 catfp.seek(0, 0);
3750 upload_file_to_internet_file(catfp, outfile);
3751 if(returnfp):
3752 catfp.seek(0, 0);
3753 return catfp;
3754 else:
3755 catfp.close();
3756 return True;
3757
3758 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3759
3760 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3761 formatspecs = FormatSpecsListToDict(formatspecs);
3762 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3763 outfile = RemoveWindowsPath(outfile);
3764 checksumtype = checksumtype.lower();
3765 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3766 checksumtype="crc32";
3767 if(checksumtype=="none"):
3768 checksumtype = "";
3769 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3770 compression = "auto";
3771 if(compression not in compressionlist and compression is None):
3772 compression = "auto";
3773 if(verbose):
3774 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3775 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3776 if(os.path.exists(outfile)):
3777 try:
3778 os.unlink(outfile);
3779 except OSError as e:
3780 pass;
3781 if(outfile=="-"):
3782 verbose = False;
3783 catfp = BytesIO();
3784 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3785 catfp = outfile;
3786 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3787 catfp = BytesIO();
3788 else:
3789 fbasename = os.path.splitext(outfile)[0];
3790 fextname = os.path.splitext(outfile)[1];
3791 if(not compresswholefile and fextname in outextlistwd):
3792 compresswholefile = True;
3793 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3794 catver = formatspecs['format_ver'];
3795 fileheaderver = str(int(catver.replace(".", "")));
3796 curinode = 0;
3797 curfid = 0;
3798 inodelist = [];
3799 inodetofile = {};
3800 filetoinode = {};
3801 inodetocatinode = {};
3802 if(infile=="-"):
3803 infile = BytesIO();
3804 if(hasattr(sys.stdin, "buffer")):
3805 shutil.copyfileobj(sys.stdin.buffer, infile);
3806 else:
3807 shutil.copyfileobj(sys.stdin, infile);
3808 infile.seek(0, 0);
3809 if(not infile):
3810 return False;
3811 infile.seek(0, 0);
3812 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3813 infile = download_file_from_internet_file(infile);
3814 infile.seek(0, 0);
3815 if(not infile):
3816 return False;
3817 infile.seek(0, 0);
3818 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3819 return False;
3820 if(not zipfile.is_zipfile(infile)):
3821 return False;
3822 try:
3823 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3824 except FileNotFoundError:
3825 return False;
3826 ziptest = zipfp.testzip();
3827 if(ziptest):
3828 VerbosePrintOut("Bad file found!");
3829 numfiles = int(len(zipfp.infolist()));
3830 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3831 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3832 if(re.findall(r"^[.|/]", member.filename)):
3833 fname = member.filename;
3834 else:
3835 fname = "./"+member.filename;
3836 zipinfo = zipfp.getinfo(member.filename);
3837 if(verbose):
3838 VerbosePrintOut(fname);
3839 if(not member.is_dir()):
3840 fpremode = int(stat.S_IFREG + 438);
3841 elif(member.is_dir()):
3842 fpremode = int(stat.S_IFDIR + 511);
3843 flinkcount = 0;
3844 ftype = 0;
3845 if(not member.is_dir()):
3846 ftype = 0;
3847 elif(member.is_dir()):
3848 ftype = 5;
3849 flinkname = "";
3850 fcurfid = format(int(curfid), 'x').lower();
3851 fcurinode = format(int(curfid), 'x').lower();
3852 curfid = curfid + 1;
3853 fdev_minor = format(int(0), 'x').lower();
3854 fdev_major = format(int(0), 'x').lower();
3855 frdev_minor = format(int(0), 'x').lower();
3856 frdev_major = format(int(0), 'x').lower();
3857 if(ftype==5):
3858 fsize = format(int("0"), 'x').lower();
3859 elif(ftype==0):
3860 fsize = format(int(member.file_size), 'x').lower();
3861 else:
3862 fsize = format(int(member.file_size), 'x').lower();
3863 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3864 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3865 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3866 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3867 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3868 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3869 if(not member.is_dir()):
3870 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3871 fchmode = stat.S_IMODE(int(stat.S_IFREG + 438));
3872 ftypemod = stat.S_IFMT(int(stat.S_IFREG + 438));
3873 elif(member.is_dir()):
3874 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3875 fchmode = stat.S_IMODE(int(stat.S_IFDIR + 511));
3876 ftypemod = stat.S_IFMT(int(stat.S_IFDIR + 511));
3877 elif(zipinfo.create_system==3):
3878 fwinattributes = format(int(0), 'x').lower();
3879 try:
3880 fmode = format(int(zipinfo.external_attr), 'x').lower();
3881 prefmode = int(zipinfo.external_attr);
3882 fchmode = stat.S_IMODE(prefmode);
3883 ftypemod = stat.S_IFMT(prefmode);
3884 except OverflowError:
3885 fmode = format(int(zipinfo.external_attr >> 16), 'x').lower();
3886 prefmode = int(zipinfo.external_attr >> 16);
3887 fchmode = stat.S_IMODE(prefmode);
3888 ftypemod = stat.S_IFMT(prefmode);
3889 else:
3890 fwinattributes = format(int(0), 'x').lower();
3891 if(not member.is_dir()):
3892 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3893 prefmode = int(stat.S_IFREG + 438);
3894 fchmode = stat.S_IMODE(prefmode);
3895 ftypemod = stat.S_IFMT(prefmode);
3896 elif(member.is_dir()):
3897 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3898 prefmode = int(stat.S_IFDIR + 511);
3899 fchmode = stat.S_IMODE(prefmode);
3900 ftypemod = stat.S_IFMT(prefmode);
3901 fcompression = "";
3902 fcsize = format(int(0), 'x').lower();
3903 try:
3904 fuid = format(int(os.getuid()), 'x').lower();
3905 except AttributeError:
3906 fuid = format(int(0), 'x').lower();
3907 except KeyError:
3908 fuid = format(int(0), 'x').lower();
3909 try:
3910 fgid = format(int(os.getgid()), 'x').lower();
3911 except AttributeError:
3912 fgid = format(int(0), 'x').lower();
3913 except KeyError:
3914 fgid = format(int(0), 'x').lower();
3915 try:
3916 import pwd;
3917 try:
3918 userinfo = pwd.getpwuid(os.getuid());
3919 funame = userinfo.pw_name;
3920 except KeyError:
3921 funame = "";
3922 except AttributeError:
3923 funame = "";
3924 except ImportError:
3925 funame = "";
3926 fgname = "";
3927 try:
3928 import grp;
3929 try:
3930 groupinfo = grp.getgrgid(os.getgid());
3931 fgname = groupinfo.gr_name;
3932 except KeyError:
3933 fgname = "";
3934 except AttributeError:
3935 fgname = "";
3936 except ImportError:
3937 fgname = "";
3938 fcontents = BytesIO();
3939 if(ftype==0):
3940 fcontents.write(zipfp.read(member.filename));
3941 if(not compresswholefile):
3942 fcontents.seek(0, 2);
3943 ucfsize = fcontents.tell();
3944 fcontents.seek(0, 0);
3945 if(compression=="auto"):
3946 ilsize = len(compressionlistalt);
3947 ilmin = 0;
3948 ilcsize = [];
3949 while(ilmin < ilsize):
3950 cfcontents = BytesIO();
3951 shutil.copyfileobj(fcontents, cfcontents);
3952 fcontents.seek(0, 0);
3953 cfcontents.seek(0, 0);
3954 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3955 cfcontents.seek(0, 2);
3956 ilcsize.append(cfcontents.tell());
3957 cfcontents.close();
3958 ilmin = ilmin + 1;
3959 ilcmin = ilcsize.index(min(ilcsize));
3960 compression = compressionlistalt[ilcmin];
3961 fcontents.seek(0, 0);
3962 cfcontents = BytesIO();
3963 shutil.copyfileobj(fcontents, cfcontents);
3964 cfcontents.seek(0, 0);
3965 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3966 cfcontents.seek(0, 2);
3967 cfsize = cfcontents.tell();
3968 if(ucfsize > cfsize):
3969 fcsize = format(int(cfsize), 'x').lower();
3970 fcompression = compression;
3971 fcontents.close();
3972 fcontents = cfcontents;
3973 if(fcompression=="none"):
3974 fcompression = "";
3975 fcontents.seek(0, 0);
3976 ftypehex = format(ftype, 'x').lower();
3977 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3978 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3979 fcontents.close();
3980 if(numfiles>0):
3981 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3982 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3983 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3984 try:
3985 catfp.flush();
3986 os.fsync(catfp.fileno());
3987 except io.UnsupportedOperation:
3988 pass;
3989 except AttributeError:
3990 pass;
3991 except OSError as e:
3992 pass;
3993 if(outfile=="-"):
3994 catfp.seek(0, 0);
3995 if(hasattr(sys.stdout, "buffer")):
3996 shutil.copyfileobj(catfp, sys.stdout.buffer);
3997 else:
3998 shutil.copyfileobj(catfp, sys.stdout);
3999 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4000 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4001 catfp.seek(0, 0);
4002 upload_file_to_internet_file(catfp, outfile);
4003 if(returnfp):
4004 catfp.seek(0, 0);
4005 return catfp;
4006 else:
4007 catfp.close();
4008 return True;
4009
4010 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
4011
4012 if(not rarfile_support):
4013 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4014 return False
4015
4016 if(rarfile_support):
4017 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4018 formatspecs = FormatSpecsListToDict(formatspecs);
4019 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4020 outfile = RemoveWindowsPath(outfile);
4021 checksumtype = checksumtype.lower();
4022 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
4023 checksumtype="crc32";
4024 if(checksumtype=="none"):
4025 checksumtype = "";
4026 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
4027 compression = "auto";
4028 if(compression not in compressionlist and compression is None):
4029 compression = "auto";
4030 if(verbose):
4031 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4032 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4033 if(os.path.exists(outfile)):
4034 try:
4035 os.unlink(outfile);
4036 except OSError as e:
4037 pass;
4038 if(outfile=="-"):
4039 verbose = False;
4040 catfp = BytesIO();
4041 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
4042 catfp = outfile;
4043 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4044 catfp = BytesIO();
4045 else:
4046 fbasename = os.path.splitext(outfile)[0];
4047 fextname = os.path.splitext(outfile)[1];
4048 if(not compresswholefile and fextname in outextlistwd):
4049 compresswholefile = True;
4050 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
4051 catver = formatspecs['format_ver'];
4052 fileheaderver = str(int(catver.replace(".", "")));
4053 curinode = 0;
4054 curfid = 0;
4055 inodelist = [];
4056 inodetofile = {};
4057 filetoinode = {};
4058 inodetocatinode = {};
4059 if(not os.path.exists(infile) or not os.path.isfile(infile)):
4060 return False;
4061 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
4062 return False;
4063 rarfp = rarfile.RarFile(infile, "r");
4064 rartest = rarfp.testrar();
4065 if(rartest):
4066 VerbosePrintOut("Bad file found!");
4067 numfiles = int(len(rarfp.infolist()));
4068 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
4069 try:
4070 catfp.flush();
4071 os.fsync(catfp.fileno());
4072 except io.UnsupportedOperation:
4073 pass;
4074 except AttributeError:
4075 pass;
4076 except OSError as e:
4077 pass;
4078 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
4079 is_unix = False;
4080 is_windows = False;
4081 if(member.host_os==rarfile.RAR_OS_UNIX):
4082 is_windows = False;
4083 try:
4084 member.external_attr
4085 is_unix = True;
4086 except AttributeError:
4087 is_unix = False;
4088 elif(member.host_os==rarfile.RAR_OS_WIN32):
4089 is_unix = False;
4090 try:
4091 member.external_attr
4092 is_windows = True;
4093 except AttributeError:
4094 is_windows = False;
4095 else:
4096 is_unix = False;
4097 is_windows = False;
4098 if(re.findall(r"^[.|/]", member.filename)):
4099 fname = member.filename;
4100 else:
4101 fname = "./"+member.filename;
4102 rarinfo = rarfp.getinfo(member.filename);
4103 if(verbose):
4104 VerbosePrintOut(fname);
4105 if(is_unix and member.external_attr !=0):
4106 fpremode = int(member.external_attr);
4107 elif(member.is_file()):
4108 fpremode = int(stat.S_IFREG + 438);
4109 elif(member.is_symlink()):
4110 fpremode = int(stat.S_IFLNK + 438);
4111 elif(member.is_dir()):
4112 fpremode = int(stat.S_IFDIR + 511);
4113 if(is_windows and member.external_attr !=0):
4114 fwinattributes = format(int(member.external_attr), 'x').lower();
4115 else:
4116 fwinattributes = format(int(0), 'x').lower();
4117 fcompression = "";
4118 fcsize = format(int(0), 'x').lower();
4119 flinkcount = 0;
4120 ftype = 0;
4121 if(member.is_file()):
4122 ftype = 0;
4123 elif(member.is_symlink()):
4124 ftype = 2;
4125 elif(member.is_dir()):
4126 ftype = 5;
4127 flinkname = "";
4128 if(ftype==2):
4129 flinkname = rarfp.read(member.filename).decode("UTF-8");
4130 fcurfid = format(int(curfid), 'x').lower();
4131 fcurinode = format(int(curfid), 'x').lower();
4132 curfid = curfid + 1;
4133 fdev_minor = format(int(0), 'x').lower();
4134 fdev_major = format(int(0), 'x').lower();
4135 frdev_minor = format(int(0), 'x').lower();
4136 frdev_major = format(int(0), 'x').lower();
4137 if(ftype==5):
4138 fsize = format(int("0"), 'x').lower();
4139 elif(ftype==0):
4140 fsize = format(int(member.file_size), 'x').lower();
4141 else:
4142 fsize = format(int(member.file_size), 'x').lower();
4143 try:
4144 if(member.atime):
4145 fatime = format(int(member.atime.timestamp()), 'x').lower();
4146 else:
4147 fatime = format(int(member.mtime.timestamp()), 'x').lower();
4148 except AttributeError:
4149 fatime = format(int(member.mtime.timestamp()), 'x').lower();
4150 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
4151 try:
4152 if(member.ctime):
4153 fctime = format(int(member.ctime.timestamp()), 'x').lower();
4154 else:
4155 fctime = format(int(member.mtime.timestamp()), 'x').lower();
4156 except AttributeError:
4157 fctime = format(int(member.mtime.timestamp()), 'x').lower();
4158 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
4159 if(is_unix and member.external_attr !=0):
4160 fmode = format(int(member.external_attr), 'x').lower();
4161 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
4162 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
4163 elif(member.is_file()):
4164 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4165 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4166 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4167 elif(member.is_symlink()):
4168 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
4169 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4170 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4171 elif(member.is_dir()):
4172 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4173 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4174 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4175 try:
4176 fuid = format(int(os.getuid()), 'x').lower();
4177 except AttributeError:
4178 fuid = format(int(0), 'x').lower();
4179 except KeyError:
4180 fuid = format(int(0), 'x').lower();
4181 try:
4182 fgid = format(int(os.getgid()), 'x').lower();
4183 except AttributeError:
4184 fgid = format(int(0), 'x').lower();
4185 except KeyError:
4186 fgid = format(int(0), 'x').lower();
4187 try:
4188 import pwd;
4189 try:
4190 userinfo = pwd.getpwuid(os.getuid());
4191 funame = userinfo.pw_name;
4192 except KeyError:
4193 funame = "";
4194 except AttributeError:
4195 funame = "";
4196 except ImportError:
4197 funame = "";
4198 fgname = "";
4199 try:
4200 import grp;
4201 try:
4202 groupinfo = grp.getgrgid(os.getgid());
4203 fgname = groupinfo.gr_name;
4204 except KeyError:
4205 fgname = "";
4206 except AttributeError:
4207 fgname = "";
4208 except ImportError:
4209 fgname = "";
4210 fcontents = BytesIO();
4211 if(ftype==0):
4212 fcontents.write(rarfp.read(member.filename));
4213 if(not compresswholefile):
4214 fcontents.seek(0, 2);
4215 ucfsize = fcontents.tell();
4216 fcontents.seek(0, 0);
4217 if(compression=="auto"):
4218 ilsize = len(compressionlistalt);
4219 ilmin = 0;
4220 ilcsize = [];
4221 while(ilmin < ilsize):
4222 cfcontents = BytesIO();
4223 shutil.copyfileobj(fcontents, cfcontents);
4224 fcontents.seek(0, 0);
4225 cfcontents.seek(0, 0);
4226 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
4227 if(cfcontents):
4228 cfcontents.seek(0, 2);
4229 ilcsize.append(cfcontents.tell());
4230 cfcontents.close();
4231 else:
4232 try:
4233 ilcsize.append(sys.maxint);
4234 except AttributeError:
4235 ilcsize.append(sys.maxsize);
4236 ilmin = ilmin + 1;
4237 ilcmin = ilcsize.index(min(ilcsize));
4238 compression = compressionlistalt[ilcmin];
4239 fcontents.seek(0, 0);
4240 cfcontents = BytesIO();
4241 shutil.copyfileobj(fcontents, cfcontents);
4242 cfcontents.seek(0, 0);
4243 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4244 cfcontents.seek(0, 2);
4245 cfsize = cfcontents.tell();
4246 if(ucfsize > cfsize):
4247 fcsize = format(int(cfsize), 'x').lower();
4248 fcompression = compression;
4249 fcontents.close();
4250 fcontents = cfcontents;
4251 if(fcompression=="none"):
4252 fcompression = "";
4253 fcontents.seek(0, 0);
4254 ftypehex = format(ftype, 'x').lower();
4255 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4256 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4257 fcontents.close();
4258 if(numfiles>0):
4259 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4260 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4261 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4262 try:
4263 catfp.flush();
4264 os.fsync(catfp.fileno());
4265 except io.UnsupportedOperation:
4266 pass;
4267 except AttributeError:
4268 pass;
4269 except OSError as e:
4270 pass;
4271 if(outfile=="-"):
4272 catfp.seek(0, 0)
4273 if(hasattr(sys.stdout, "buffer")):
4274 shutil.copyfileobj(catfp, sys.stdout.buffer);
4275 else:
4276 shutil.copyfileobj(catfp, sys.stdout);
4277 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4278 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4279 catfp.seek(0, 0);
4280 upload_file_to_internet_file(catfp, outfile);
4281 if(returnfp):
4282 catfp.seek(0, 0)
4283 return catfp
4284 else:
4285 catfp.close()
4286 return True;
4287
4288 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
4289
4290 if(not py7zr_support):
4291 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4292 return False
4293
4294 if(py7zr_support):
4295 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4296 formatspecs = FormatSpecsListToDict(formatspecs);
4297 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4298 outfile = RemoveWindowsPath(outfile);
4299 checksumtype = checksumtype.lower();
4300 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
4301 checksumtype="crc32";
4302 if(checksumtype=="none"):
4303 checksumtype = "";
4304 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
4305 compression = "auto";
4306 if(compression not in compressionlist and compression is None):
4307 compression = "auto";
4308 if(verbose):
4309 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4310 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
4311 if(os.path.exists(outfile)):
4312 try:
4313 os.unlink(outfile);
4314 except OSError as e:
4315 pass;
4316 if(outfile=="-"):
4317 verbose = False;
4318 catfp = BytesIO();
4319 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
4320 catfp = outfile;
4321 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4322 catfp = BytesIO();
4323 else:
4324 fbasename = os.path.splitext(outfile)[0];
4325 fextname = os.path.splitext(outfile)[1];
4326 if(not compresswholefile and fextname in outextlistwd):
4327 compresswholefile = True;
4328 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
4329 catver = formatspecs['format_ver'];
4330 fileheaderver = str(int(catver.replace(".", "")));
4331 curinode = 0;
4332 curfid = 0;
4333 inodelist = [];
4334 inodetofile = {};
4335 filetoinode = {};
4336 inodetocatinode = {};
4337 if(not os.path.exists(infile) or not os.path.isfile(infile)):
4338 return False;
4339 szpfp = py7zr.SevenZipFile(infile, mode="r");
4340 file_content = szpfp.readall();
4341 #sztest = szpfp.testzip();
4342 sztestalt = szpfp.test();
4343 if(sztestalt):
4344 VerbosePrintOut("Bad file found!");
4345 numfiles = int(len(szpfp.list()));
4346 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
4347 for member in sorted(szpfp.list(), key=lambda x: x.filename):
4348 if(re.findall(r"^[.|/]", member.filename)):
4349 fname = member.filename;
4350 else:
4351 fname = "./"+member.filename;
4352 if(verbose):
4353 VerbosePrintOut(fname);
4354 if(not member.is_directory):
4355 fpremode = int(stat.S_IFREG + 438);
4356 elif(member.is_directory):
4357 fpremode = int(stat.S_IFDIR + 511);
4358 fwinattributes = format(int(0), 'x').lower();
4359 fcompression = "";
4360 fcsize = format(int(0), 'x').lower();
4361 flinkcount = 0;
4362 ftype = 0;
4363 if(member.is_directory):
4364 ftype = 5;
4365 else:
4366 ftype = 0;
4367 flinkname = "";
4368 fcurfid = format(int(curfid), 'x').lower();
4369 fcurinode = format(int(curfid), 'x').lower();
4370 curfid = curfid + 1;
4371 fdev_minor = format(int(0), 'x').lower();
4372 fdev_major = format(int(0), 'x').lower();
4373 frdev_minor = format(int(0), 'x').lower();
4374 frdev_major = format(int(0), 'x').lower();
4375 if(ftype==5):
4376 fsize = format(int("0"), 'x').lower();
4377 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
4378 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
4379 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
4380 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
4381 if(member.is_directory):
4382 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4383 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4384 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4385 else:
4386 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4387 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4388 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4389 try:
4390 fuid = format(int(os.getuid()), 'x').lower();
4391 except AttributeError:
4392 fuid = format(int(0), 'x').lower();
4393 except KeyError:
4394 fuid = format(int(0), 'x').lower();
4395 try:
4396 fgid = format(int(os.getgid()), 'x').lower();
4397 except AttributeError:
4398 fgid = format(int(0), 'x').lower();
4399 except KeyError:
4400 fgid = format(int(0), 'x').lower();
4401 try:
4402 import pwd;
4403 try:
4404 userinfo = pwd.getpwuid(os.getuid());
4405 funame = userinfo.pw_name;
4406 except KeyError:
4407 funame = "";
4408 except AttributeError:
4409 funame = "";
4410 except ImportError:
4411 funame = "";
4412 fgname = "";
4413 try:
4414 import grp;
4415 try:
4416 groupinfo = grp.getgrgid(os.getgid());
4417 fgname = groupinfo.gr_name;
4418 except KeyError:
4419 fgname = "";
4420 except AttributeError:
4421 fgname = "";
4422 except ImportError:
4423 fgname = "";
4424 fcontents = BytesIO();
4425 if(ftype==0):
4426 fcontents.write(file_content[member.filename].read());
4427 fsize = format(fcontents.tell(), 'x').lower();
4428 file_content[member.filename].close();
4429 if(not compresswholefile):
4430 fcontents.seek(0, 2);
4431 ucfsize = fcontents.tell();
4432 fcontents.seek(0, 0);
4433 if(compression=="auto"):
4434 ilsize = len(compressionlistalt);
4435 ilmin = 0;
4436 ilcsize = [];
4437 while(ilmin < ilsize):
4438 cfcontents = BytesIO();
4439 shutil.copyfileobj(fcontents, cfcontents);
4440 fcontents.seek(0, 0);
4441 cfcontents.seek(0, 0);
4442 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
4443 if(cfcontents):
4444 cfcontents.seek(0, 2);
4445 ilcsize.append(cfcontents.tell());
4446 cfcontents.close();
4447 else:
4448 try:
4449 ilcsize.append(sys.maxint);
4450 except AttributeError:
4451 ilcsize.append(sys.maxsize);
4452 ilmin = ilmin + 1;
4453 ilcmin = ilcsize.index(min(ilcsize));
4454 compression = compressionlistalt[ilcmin];
4455 fcontents.seek(0, 0);
4456 cfcontents = BytesIO();
4457 shutil.copyfileobj(fcontents, cfcontents);
4458 cfcontents.seek(0, 0);
4459 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4460 cfcontents.seek(0, 2);
4461 cfsize = cfcontents.tell();
4462 if(ucfsize > cfsize):
4463 fcsize = format(int(cfsize), 'x').lower();
4464 fcompression = compression;
4465 fcontents.close();
4466 fcontents = cfcontents;
4467 if(fcompression=="none"):
4468 fcompression = "";
4469 fcontents.seek(0, 0);
4470 ftypehex = format(ftype, 'x').lower();
4471 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4472 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4473 fcontents.close();
4474 if(numfiles>0):
4475 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4476 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4477 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4478 try:
4479 catfp.flush();
4480 os.fsync(catfp.fileno());
4481 except io.UnsupportedOperation:
4482 pass;
4483 except AttributeError:
4484 pass;
4485 except OSError as e:
4486 pass;
4487 if(outfile=="-"):
4488 catfp.seek(0, 0);
4489 if(hasattr(sys.stdout, "buffer")):
4490 shutil.copyfileobj(catfp, sys.stdout.buffer);
4491 else:
4492 shutil.copyfileobj(catfp, sys.stdout);
4493 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4494 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4495 catfp.seek(0, 0);
4496 upload_file_to_internet_file(catfp, outfile);
4497 if(returnfp):
4498 catfp.seek(0, 0)
4499 return catfp
4500 else:
4501 catfp.close()
4502 return True;
4503
4504 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4505
4506 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4507 formatspecs = FormatSpecsListToDict(formatspecs);
4508 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4509 if(verbose):
4510 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4511 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4512 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4513 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4514 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4515 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4516 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4517 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4518 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4519 elif(checkcompressfile=="catfile"):
4520 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4521 else:
4522 return False;
4523 return False;
4524
4525 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4526
4527 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4528 formatspecs = FormatSpecsListToDict(formatspecs);
4529 if(hasattr(infile, "read") or hasattr(infile, "write")):
4530 catfp = infile;
4531 catfp.seek(0, 0);
4532 catfp = UncompressArchiveFile(catfp, formatspecs);
4533 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4534 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4535 return TarFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4536 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4537 return ZipFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4538 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4539 return RarFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4540 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4541 return SevenZipFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4542 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4543 return False;
4544 if(not catfp):
4545 return False;
4546 catfp.seek(0, 0);
4547 elif(infile=="-"):
4548 catfp = BytesIO();
4549 if(hasattr(sys.stdin, "buffer")):
4550 shutil.copyfileobj(sys.stdin.buffer, catfp);
4551 else:
4552 shutil.copyfileobj(sys.stdin, catfp);
4553 catfp.seek(0, 0);
4554 catfp = UncompressArchiveFile(catfp, formatspecs);
4555 if(not catfp):
4556 return False;
4557 catfp.seek(0, 0);
4558 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4559 catfp = download_file_from_internet_file(infile);
4560 catfp.seek(0, 0);
4561 catfp = UncompressArchiveFile(catfp, formatspecs);
4562 if(not catfp):
4563 return False;
4564 catfp.seek(0, 0);
4565 else:
4566 infile = RemoveWindowsPath(infile);
4567 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4568 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4569 return TarFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4570 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4571 return ZipFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4572 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4573 return RarFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4574 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4575 return SevenZipFileToArray(infile, seekto, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4576 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4577 return False;
4578 compresscheck = CheckCompressionType(infile, formatspecs, True);
4579 if(not compresscheck):
4580 fextname = os.path.splitext(infile)[1];
4581 if(fextname==".gz"):
4582 compresscheck = "gzip";
4583 elif(fextname==".bz2"):
4584 compresscheck = "bzip2";
4585 elif(fextname==".zst"):
4586 compresscheck = "zstd";
4587 elif(fextname==".lz4" or fextname==".clz4"):
4588 compresscheck = "lz4";
4589 elif(fextname==".lzo" or fextname==".lzop"):
4590 compresscheck = "lzo";
4591 elif(fextname==".lzma"):
4592 compresscheck = "lzma";
4593 elif(fextname==".xz"):
4594 compresscheck = "xz";
4595 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
4596 compresscheck = "zlib";
4597 else:
4598 return False;
4599 if(not compresscheck):
4600 return False;
4601 catfp = UncompressFile(infile, formatspecs, "rb");
4602 '''
4603 try:
4604 catfp.seek(0, 2);
4605 except OSError:
4606 SeekToEndOfFile(catfp);
4607 except ValueError:
4608 SeekToEndOfFile(catfp);
4609 CatSize = catfp.tell();
4610 CatSizeEnd = CatSize;
4611 '''
4612 try:
4613 catfp.seek(0, 0);
4614 except OSError:
4615 return False;
4616 except ValueError:
4617 return False;
4618 curloc = catfp.tell();
4619 if(curloc>0):
4620 catfp.seek(0, 0);
4621 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4622 if(curloc>0):
4623 catfp.seek(curloc, 0);
4624 catstring = catheader[0];
4625 catversion = re.findall(r"([\d]+)", catstring);
4626 fprenumfiles = catheader[1];
4627 fnumfiles = int(fprenumfiles, 16);
4628 fprechecksumtype = catheader[2];
4629 fprechecksum = catheader[3];
4630 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4631 fnumfileshex = format(int(fnumfiles), 'x').lower();
4632 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4633 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4634 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4635 fheadtell = len(fileheader);
4636 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4637 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4638 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4639 return False;
4640 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4641 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4642 if(seekto>=fnumfiles):
4643 seekto = fnumfiles - 1;
4644 if(seekto<0):
4645 seekto = 0;
4646 if(seekto>=0):
4647 il = -1;
4648 while(il < seekto):
4649 prefhstart = catfp.tell();
4650 if(formatspecs['new_style']):
4651 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4652 else:
4653 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4654 if(len(preheaderdata)==0):
4655 break;
4656 prefheadsize = int(preheaderdata[0], 16);
4657 prefnumfields = int(preheaderdata[1], 16);
4658 preftype = int(preheaderdata[2], 16);
4659 if(re.findall(r"^[.|/]", preheaderdata[3])):
4660 prefname = preheaderdata[3];
4661 else:
4662 prefname = "./"+preheaderdata[3];
4663 prefbasedir = os.path.dirname(prefname);
4664 preflinkname = preheaderdata[4];
4665 prefsize = int(preheaderdata[5], 16);
4666 prefatime = int(preheaderdata[6], 16);
4667 prefmtime = int(preheaderdata[7], 16);
4668 prefctime = int(preheaderdata[8], 16);
4669 prefbtime = int(preheaderdata[9], 16);
4670 prefmode = int(preheaderdata[10], 16);
4671 prefchmode = stat.S_IMODE(prefmode);
4672 preftypemod = stat.S_IFMT(prefmode);
4673 prefwinattributes = int(preheaderdata[11], 16);
4674 prefcompression = preheaderdata[12];
4675 prefcsize = int(preheaderdata[13], 16);
4676 prefuid = int(preheaderdata[14], 16);
4677 prefuname = preheaderdata[15];
4678 prefgid = int(preheaderdata[16], 16);
4679 prefgname = preheaderdata[17];
4680 fid = int(preheaderdata[18], 16);
4681 finode = int(preheaderdata[19], 16);
4682 flinkcount = int(preheaderdata[20], 16);
4683 prefdev_minor = int(preheaderdata[21], 16);
4684 prefdev_major = int(preheaderdata[22], 16);
4685 prefrdev_minor = int(preheaderdata[23], 16);
4686 prefrdev_major = int(preheaderdata[24], 16);
4687 prefseeknextfile = preheaderdata[25];
4688 prefextrasize = int(preheaderdata[26], 16);
4689 prefextrafields = int(preheaderdata[27], 16);
4690 extrafieldslist = [];
4691 extrastart = 28;
4692 extraend = extrastart + prefextrafields;
4693 extrafieldslist = [];
4694 if(extrastart<extraend):
4695 extrafieldslist.append(preheaderdata[extrastart]);
4696 extrastart = extrastart + 1;
4697 prefcs = preheaderdata[-2].lower();
4698 prenewfcs = preheaderdata[-1].lower();
4699 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4700 if(prefcs!=prenewfcs and not skipchecksum):
4701 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4702 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4703 return False;
4704 valid_archive = False;
4705 invalid_archive = True;
4706 prefhend = catfp.tell() - 1;
4707 prefcontentstart = catfp.tell();
4708 prefcontents = "";
4709 pyhascontents = False;
4710 if(prefsize>0):
4711 if(prefcompression):
4712 prefcontents = catfp.read(prefsize);
4713 else:
4714 prefcontents = catfp.read(prefcsize);
4715 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4716 pyhascontents = True;
4717 if(prefccs!=prenewfccs and not skipchecksum):
4718 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4719 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4720 return False;
4721 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4722 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4723 if(abs(fseeknextasnum)==0):
4724 pass;
4725 catfp.seek(fseeknextasnum, 1);
4726 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4727 fseeknextasnum = int(prefseeknextfile);
4728 if(abs(fseeknextasnum)==0):
4729 pass;
4730 catfp.seek(fseeknextasnum, 1);
4731 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4732 fseeknextasnum = int(prefseeknextfile);
4733 if(abs(fseeknextasnum)==0):
4734 pass;
4735 catfp.seek(fseeknextasnum, 0);
4736 else:
4737 return False;
4738 il = il + 1;
4739 catfp.seek(seekstart, 0);
4740 fileidnum = il;
4741 catfheadsize = int(preheaderdata[0], 16);
4742 catfnumfields = int(preheaderdata[1], 16);
4743 catftype = int(preheaderdata[2], 16);
4744 if(re.findall(r"^[.|/]", preheaderdata[3])):
4745 catfname = preheaderdata[3];
4746 else:
4747 catfname = "./"+preheaderdata[3];
4748 catflinkname = preheaderdata[4];
4749 catfsize = int(preheaderdata[5], 16);
4750 catfbasedir = os.path.dirname(catfname);
4751 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4752 if(returnfp):
4753 catlist.update({'catfp': catfp});
4754 else:
4755 catfp.close();
4756 return catlist;
4757
4758 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4759
4760 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4761 formatspecs = FormatSpecsListToDict(formatspecs);
4762 if(hasattr(infile, "read") or hasattr(infile, "write")):
4763 catfp = infile;
4764 catfp.seek(0, 0);
4765 catfp = UncompressArchiveFile(catfp, formatspecs);
4766 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4767 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4768 return TarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4769 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4770 return ZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4771 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4772 return RarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4773 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4774 return SevenZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4775 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4776 return False;
4777 if(not catfp):
4778 return False;
4779 catfp.seek(0, 0);
4780 elif(infile=="-"):
4781 catfp = BytesIO();
4782 if(hasattr(sys.stdin, "buffer")):
4783 shutil.copyfileobj(sys.stdin.buffer, catfp);
4784 else:
4785 shutil.copyfileobj(sys.stdin, catfp);
4786 catfp.seek(0, 0);
4787 catfp = UncompressArchiveFile(catfp, formatspecs);
4788 if(not catfp):
4789 return False;
4790 catfp.seek(0, 0);
4791 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4792 catfp = download_file_from_internet_file(infile);
4793 catfp = UncompressArchiveFile(catfp, formatspecs);
4794 catfp.seek(0, 0);
4795 if(not catfp):
4796 return False;
4797 catfp.seek(0, 0);
4798 else:
4799 infile = RemoveWindowsPath(infile);
4800 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4801 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4802 return TarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4803 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4804 return ZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4805 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4806 return RarFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4807 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4808 return SevenZipFileToArray(infile, 0, 0, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
4809 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4810 return False;
4811 compresscheck = CheckCompressionType(infile, formatspecs, True);
4812 if(not compresscheck):
4813 fextname = os.path.splitext(infile)[1];
4814 if(fextname==".gz"):
4815 compresscheck = "gzip";
4816 elif(fextname==".bz2"):
4817 compresscheck = "bzip2";
4818 elif(fextname==".zst"):
4819 compresscheck = "zstd";
4820 elif(fextname==".lz4" or fextname==".clz4"):
4821 compresscheck = "lz4";
4822 elif(fextname==".lzo" or fextname==".lzop"):
4823 compresscheck = "lzo";
4824 elif(fextname==".lzma"):
4825 compresscheck = "lzma";
4826 elif(fextname==".xz"):
4827 compresscheck = "xz";
4828 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
4829 compresscheck = "zlib";
4830 else:
4831 return False;
4832 if(not compresscheck):
4833 return False;
4834 catfp = UncompressFile(infile, formatspecs, "rb");
4835 '''
4836 try:
4837 catfp.seek(0, 2);
4838 except OSError:
4839 SeekToEndOfFile(catfp);
4840 except ValueError:
4841 SeekToEndOfFile(catfp);
4842 CatSize = catfp.tell();
4843 CatSizeEnd = CatSize;
4844 '''
4845 try:
4846 catfp.seek(0, 0);
4847 except OSError:
4848 return False;
4849 except ValueError:
4850 return False;
4851 curloc = catfp.tell();
4852 if(curloc>0):
4853 catfp.seek(0, 0);
4854 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4855 if(curloc>0):
4856 catfp.seek(curloc, 0);
4857 catstring = catheader[0];
4858 catversion = re.findall(r"([\d]+)", catstring);
4859 fprenumfiles = catheader[1];
4860 fnumfiles = int(fprenumfiles, 16);
4861 fprechecksumtype = catheader[2];
4862 fprechecksum = catheader[3];
4863 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4864 fnumfileshex = format(int(fnumfiles), 'x').lower();
4865 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4866 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4867 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4868 fheadtell = len(fileheader);
4869 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4870 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4871 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4872 return False;
4873 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4874 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4875 seekto = fnumfiles - 1
4876 filefound = False;
4877 if(seekto>=0):
4878 il = -1;
4879 while(il < seekto):
4880 prefhstart = catfp.tell();
4881 if(formatspecs['new_style']):
4882 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4883 else:
4884 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4885 if(len(preheaderdata)==0):
4886 break;
4887 prefheadsize = int(preheaderdata[0], 16);
4888 prefnumfields = int(preheaderdata[1], 16);
4889 preftype = int(preheaderdata[2], 16);
4890 if(re.findall(r"^[.|/]", preheaderdata[3])):
4891 prefname = preheaderdata[3];
4892 else:
4893 prefname = "./"+preheaderdata[3];
4894 prefbasedir = os.path.dirname(prefname);
4895 preflinkname = preheaderdata[4];
4896 prefsize = int(preheaderdata[5], 16);
4897 prefatime = int(preheaderdata[6], 16);
4898 prefmtime = int(preheaderdata[7], 16);
4899 prefctime = int(preheaderdata[8], 16);
4900 prefbtime = int(preheaderdata[9], 16);
4901 prefmode = int(preheaderdata[10], 16);
4902 prefchmode = stat.S_IMODE(prefmode);
4903 preftypemod = stat.S_IFMT(prefmode);
4904 prefwinattributes = int(preheaderdata[11], 16);
4905 prefcompression = preheaderdata[12];
4906 prefcsize = int(preheaderdata[13], 16);
4907 prefuid = int(preheaderdata[14], 16);
4908 prefuname = preheaderdata[15];
4909 prefgid = int(preheaderdata[16], 16);
4910 prefgname = preheaderdata[17];
4911 fid = int(preheaderdata[18], 16);
4912 finode = int(preheaderdata[19], 16);
4913 flinkcount = int(preheaderdata[20], 16);
4914 prefdev_minor = int(preheaderdata[21], 16);
4915 prefdev_major = int(preheaderdata[22], 16);
4916 prefrdev_minor = int(preheaderdata[23], 16);
4917 prefrdev_major = int(preheaderdata[24], 16);
4918 prefseeknextfile = preheaderdata[25];
4919 prefextrasize = int(preheaderdata[26], 16);
4920 prefextrafields = int(preheaderdata[27], 16);
4921 extrafieldslist = [];
4922 extrastart = 28;
4923 extraend = extrastart + prefextrafields;
4924 extrafieldslist = [];
4925 if(extrastart<extraend):
4926 extrafieldslist.append(preheaderdata[extrastart]);
4927 extrastart = extrastart + 1;
4928 prefcs = preheaderdata[-2].lower();
4929 prenewfcs = preheaderdata[-1].lower();
4930 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4931 if(prefcs!=prenewfcs and not skipchecksum):
4932 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4933 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4934 return False;
4935 valid_archive = False;
4936 invalid_archive = True;
4937 prefhend = catfp.tell() - 1;
4938 prefcontentstart = catfp.tell();
4939 prefcontents = "";
4940 pyhascontents = False;
4941 if(prefsize>0):
4942 if(prefcompression):
4943 prefcontents = catfp.read(prefsize);
4944 else:
4945 prefcontents = catfp.read(prefcsize);
4946 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4947 pyhascontents = True;
4948 if(prefccs!=prenewfccs and not skipchecksum):
4949 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4950 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4951 return False;
4952 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4953 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4954 if(abs(fseeknextasnum)==0):
4955 pass;
4956 catfp.seek(fseeknextasnum, 1);
4957 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4958 fseeknextasnum = int(prefseeknextfile);
4959 if(abs(fseeknextasnum)==0):
4960 pass;
4961 catfp.seek(fseeknextasnum, 1);
4962 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4963 fseeknextasnum = int(prefseeknextfile);
4964 if(abs(fseeknextasnum)==0):
4965 pass;
4966 catfp.seek(fseeknextasnum, 0);
4967 else:
4968 return False;
4969 il = il + 1;
4970 filefound = False;
4971 prefname = preheaderdata[2];
4972 if(re.findall(r"^[.|/]", preheaderdata[2])):
4973 prefname = preheaderdata[2];
4974 else:
4975 prefname = "./"+preheaderdata[2];
4976 if(prefname==seekfile):
4977 filefound = True;
4978 break;
4979 catfp.seek(seekstart, 0);
4980 fileidnum = il;
4981 catfheadsize = int(preheaderdata[0], 16);
4982 catfnumfields = int(preheaderdata[1], 16);
4983 catftype = int(preheaderdata[2], 16);
4984 if(re.findall(r"^[.|/]", preheaderdata[3])):
4985 catfname = preheaderdata[3];
4986 else:
4987 catfname = "./"+preheaderdata[3];
4988 catflinkname = preheaderdata[4];
4989 catfsize = int(preheaderdata[5], 16);
4990 catfbasedir = os.path.dirname(catfname);
4991 if(filefound):
4992 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4993 else:
4994 return False;
4995 if(returnfp):
4996 catlist.update({'catfp': catfp});
4997 else:
4998 catfp.close();
4999 return catlist;
5000
5001 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
5002
5003 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
5004 formatspecs = FormatSpecsListToDict(formatspecs);
5005 if(verbose):
5006 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5007 if(hasattr(infile, "read") or hasattr(infile, "write")):
5008 catfp = infile;
5009 catfp.seek(0, 0);
5010 catfp = UncompressArchiveFile(catfp, formatspecs);
5011 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
5012 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5013 return TarFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5014 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5015 return ZipFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5016 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5017 return RarFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5018 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5019 return SevenZipFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5020 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5021 return False;
5022 if(not catfp):
5023 return False;
5024 catfp.seek(0, 0);
5025 elif(infile=="-"):
5026 catfp = BytesIO();
5027 if(hasattr(sys.stdin, "buffer")):
5028 shutil.copyfileobj(sys.stdin.buffer, catfp);
5029 else:
5030 shutil.copyfileobj(sys.stdin, catfp);
5031 catfp.seek(0, 0);
5032 catfp = UncompressArchiveFile(catfp, formatspecs);
5033 if(not catfp):
5034 return False;
5035 catfp.seek(0, 0);
5036 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5037 catfp = download_file_from_internet_file(infile);
5038 catfp = UncompressArchiveFile(catfp, formatspecs);
5039 catfp.seek(0, 0);
5040 if(not catfp):
5041 return False;
5042 catfp.seek(0, 0);
5043 else:
5044 infile = RemoveWindowsPath(infile);
5045 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5046 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5047 return TarFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5048 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5049 return ZipFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5050 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5051 return RarFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5052 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5053 return SevenZipFileToArray(infile, 0, 0, False, True, False, formatspecs, returnfp);
5054 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5055 return False;
5056 compresscheck = CheckCompressionType(infile, formatspecs, True);
5057 if(not compresscheck):
5058 fextname = os.path.splitext(infile)[1];
5059 if(fextname==".gz"):
5060 compresscheck = "gzip";
5061 elif(fextname==".bz2"):
5062 compresscheck = "bzip2";
5063 elif(fextname==".zst"):
5064 compresscheck = "zstd";
5065 elif(fextname==".lz4" or fextname==".clz4"):
5066 compresscheck = "lz4";
5067 elif(fextname==".lzo" or fextname==".lzop"):
5068 compresscheck = "lzo";
5069 elif(fextname==".lzma"):
5070 compresscheck = "lzma";
5071 elif(fextname==".xz"):
5072 compresscheck = "xz";
5073 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
5074 compresscheck = "zlib";
5075 else:
5076 return False;
5077 if(not compresscheck):
5078 return False;
5079 catfp = UncompressFile(infile, formatspecs, "rb");
5080 '''
5081 try:
5082 catfp.seek(0, 2);
5083 except OSError:
5084 SeekToEndOfFile(catfp);
5085 except ValueError:
5086 SeekToEndOfFile(catfp);
5087 CatSize = catfp.tell();
5088 CatSizeEnd = CatSize;
5089 '''
5090 try:
5091 catfp.seek(0, 0);
5092 except OSError:
5093 return False;
5094 except ValueError:
5095 return False;
5096 curloc = catfp.tell();
5097 if(curloc>0):
5098 catfp.seek(0, 0);
5099 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
5100 if(curloc>0):
5101 catfp.seek(curloc, 0);
5102 catstring = catheader[0];
5103 catversion = re.findall(r"([\d]+)", catstring);
5104 fprenumfiles = catheader[1];
5105 fnumfiles = int(fprenumfiles, 16);
5106 fprechecksumtype = catheader[2];
5107 fprechecksum = catheader[3];
5108 il = 0;
5109 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
5110 fnumfileshex = format(int(fnumfiles), 'x').lower();
5111 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
5112 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5113 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5114 valid_archive = True;
5115 invalid_archive = False;
5116 if(verbose):
5117 if(hasattr(infile, "read") or hasattr(infile, "write")):
5118 try:
5119 VerbosePrintOut(infile.name);
5120 except AttributeError:
5121 VerbosePrintOut(infile);
5122 else:
5123 VerbosePrintOut(infile);
5124 VerbosePrintOut("Number of Records " + str(fnumfiles));
5125 if(fprechecksum==catfileheadercshex):
5126 if(verbose):
5127 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
5128 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
5129 else:
5130 if(verbose):
5131 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
5132 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
5133 valid_archive = False;
5134 invalid_archive = True;
5135 if(verbose):
5136 VerbosePrintOut("");
5137 while(il<fnumfiles):
5138 catfhstart = catfp.tell();
5139 if(formatspecs['new_style']):
5140 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
5141 else:
5142 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
5143 if(len(catheaderdata)==0):
5144 break;
5145 catfheadsize = int(catheaderdata[0], 16);
5146 catfnumfields = int(catheaderdata[1], 16);
5147 catftype = int(catheaderdata[2], 16);
5148 if(re.findall(r"^[.|/]", catheaderdata[3])):
5149 catfname = catheaderdata[3];
5150 else:
5151 catfname = "./"+catheaderdata[3];
5152 catfbasedir = os.path.dirname(catfname);
5153 catflinkname = catheaderdata[4];
5154 catfsize = int(catheaderdata[5], 16);
5155 catfatime = int(catheaderdata[6], 16);
5156 catfmtime = int(catheaderdata[7], 16);
5157 catfctime = int(catheaderdata[8], 16);
5158 catfbtime = int(catheaderdata[9], 16);
5159 catfmode = int(catheaderdata[10], 16);
5160 catfchmode = stat.S_IMODE(catfmode);
5161 catftypemod = stat.S_IFMT(catfmode);
5162 catfwinattributes = int(catheaderdata[11], 16);
5163 catfcompression = catheaderdata[12];
5164 catfcsize = int(catheaderdata[13], 16);
5165 catfuid = int(catheaderdata[14], 16);
5166 catfuname = catheaderdata[15];
5167 catfgid = int(catheaderdata[16], 16);
5168 catfgname = catheaderdata[17];
5169 fid = int(catheaderdata[18], 16);
5170 finode = int(catheaderdata[19], 16);
5171 flinkcount = int(catheaderdata[20], 16);
5172 catfdev_minor = int(catheaderdata[21], 16);
5173 catfdev_major = int(catheaderdata[22], 16);
5174 catfrdev_minor = int(catheaderdata[23], 16);
5175 catfrdev_major = int(catheaderdata[24], 16);
5176 catfseeknextfile = catheaderdata[25];
5177 catfextrasize = int(catheaderdata[26], 16);
5178 catfextrafields = int(catheaderdata[27], 16);
5179 extrafieldslist = [];
5180 extrastart = 28;
5181 extraend = extrastart + catfextrafields;
5182 extrafieldslist = [];
5183 if(extrastart<extraend):
5184 extrafieldslist.append(catheaderdata[extrastart]);
5185 extrastart = extrastart + 1;
5186 catfcs = catheaderdata[-2].lower();
5187 catfccs = catheaderdata[-1].lower();
5188 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5189 if(verbose):
5190 VerbosePrintOut(catfname);
5191 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
5192 if(catfcs==catnewfcs):
5193 if(verbose):
5194 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
5195 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
5196 else:
5197 if(verbose):
5198 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
5199 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5200 valid_archive = False;
5201 invalid_archive = True;
5202 catfhend = catfp.tell() - 1;
5203 catfcontentstart = catfp.tell();
5204 catfcontents = "";
5205 pyhascontents = False;
5206 if(catfsize>0):
5207 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5208 catfcontents = catfp.read(catfsize);
5209 else:
5210 catfcontents = catfp.read(catfcsize);
5211 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
5212 pyhascontents = True;
5213 if(catfccs==catnewfccs):
5214 if(verbose):
5215 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
5216 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
5217 else:
5218 if(verbose):
5219 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
5220 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5221 valid_archive = False;
5222 invalid_archive = True;
5223 if(verbose):
5224 VerbosePrintOut("");
5225 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5226 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5227 if(abs(fseeknextasnum)==0):
5228 pass;
5229 catfp.seek(fseeknextasnum, 1);
5230 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5231 fseeknextasnum = int(catfseeknextfile);
5232 if(abs(fseeknextasnum)==0):
5233 pass;
5234 catfp.seek(fseeknextasnum, 1);
5235 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5236 fseeknextasnum = int(catfseeknextfile);
5237 if(abs(fseeknextasnum)==0):
5238 pass;
5239 catfp.seek(fseeknextasnum, 0);
5240 else:
5241 return False;
5242 il = il + 1;
5243 if(valid_archive):
5244 if(returnfp):
5245 return catfp;
5246 else:
5247 catfp.close();
5248 return True;
5249 else:
5250 catfp.close();
5251 return False;
5252
5253 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
5254
5255 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5256 formatspecs = FormatSpecsListToDict(formatspecs);
5257 if(hasattr(infile, "read") or hasattr(infile, "write")):
5258 catfp = infile;
5259 catfp.seek(0, 0);
5260 catfp = UncompressArchiveFile(catfp, formatspecs);
5261 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
5262 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5263 return TarFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5264 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5265 return ZipFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5266 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5267 return RarFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5268 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5269 return SevenZipFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5270 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5271 return False;
5272 if(not catfp):
5273 return False;
5274 catfp.seek(0, 0);
5275 elif(infile=="-"):
5276 catfp = BytesIO();
5277 if(hasattr(sys.stdin, "buffer")):
5278 shutil.copyfileobj(sys.stdin.buffer, catfp);
5279 else:
5280 shutil.copyfileobj(sys.stdin, catfp);
5281 catfp.seek(0, 0);
5282 catfp = UncompressArchiveFile(catfp, formatspecs);
5283 if(not catfp):
5284 return False;
5285 catfp.seek(0, 0);
5286 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5287 catfp = download_file_from_internet_file(infile);
5288 catfp = UncompressArchiveFile(catfp, formatspecs);
5289 catfp.seek(0, 0);
5290 if(not catfp):
5291 return False;
5292 catfp.seek(0, 0);
5293 else:
5294 infile = RemoveWindowsPath(infile);
5295 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5296 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5297 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5298 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5299 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5300 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5301 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5302 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5303 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5304 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
5305 return False;
5306 compresscheck = CheckCompressionType(infile, formatspecs, True);
5307 if(not compresscheck):
5308 fextname = os.path.splitext(infile)[1];
5309 if(fextname==".gz"):
5310 compresscheck = "gzip";
5311 elif(fextname==".bz2"):
5312 compresscheck = "bzip2";
5313 elif(fextname==".zst"):
5314 compresscheck = "zstd";
5315 elif(fextname==".lz4" or fextname==".clz4"):
5316 compresscheck = "lz4";
5317 elif(fextname==".lzo" or fextname==".lzop"):
5318 compresscheck = "lzo";
5319 elif(fextname==".lzma"):
5320 compresscheck = "lzma";
5321 elif(fextname==".xz"):
5322 compresscheck = "xz";
5323 elif(fextname==".zz" or fextname==".zl" or fextname==".zlib"):
5324 compresscheck = "zlib";
5325 else:
5326 return False;
5327 if(not compresscheck):
5328 return False;
5329 catfp = UncompressFile(infile, formatspecs, "rb");
5330 '''
5331 try:
5332 catfp.seek(0, 2);
5333 except OSError:
5334 SeekToEndOfFile(catfp);
5335 except ValueError:
5336 SeekToEndOfFile(catfp);
5337 CatSize = catfp.tell();
5338 CatSizeEnd = CatSize;
5339 '''
5340 try:
5341 catfp.seek(0, 0);
5342 except OSError:
5343 return False;
5344 except ValueError:
5345 return False;
5346 curloc = catfp.tell();
5347 if(curloc>0):
5348 catfp.seek(0, 0);
5349 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
5350 if(curloc>0):
5351 catfp.seek(curloc, 0);
5352 catstring = catheader[0];
5353 catversion = re.findall(r"([\d]+)", catstring);
5354 fprenumfiles = catheader[1];
5355 fnumfiles = int(fprenumfiles, 16);
5356 fprechecksumtype = catheader[2];
5357 fprechecksum = catheader[3];
5358 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
5359 fnumfileshex = format(int(fnumfiles), 'x').lower();
5360 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
5361 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5362 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5363 fheadtell = len(fileheader);
5364 if(fprechecksum!=catfileheadercshex and not skipchecksum):
5365 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
5366 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
5367 return False;
5368 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
5369 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
5370 if(seekstart<0 and seekstart>fnumfiles):
5371 seekstart = 0;
5372 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
5373 seekend = fnumfiles;
5374 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
5375 seekend = fnumfiles - abs(seekend);
5376 if(seekstart>0):
5377 il = 0;
5378 while(il < seekstart):
5379 prefhstart = catfp.tell();
5380 if(formatspecs['new_style']):
5381 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
5382 else:
5383 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
5384 if(len(preheaderdata)==0):
5385 break;
5386 prefheadsize = int(preheaderdata[0], 16);
5387 prefnumfields = int(preheaderdata[1], 16);
5388 if(re.findall(r"^[.|/]", preheaderdata[3])):
5389 prefname = preheaderdata[3];
5390 else:
5391 prefname = "./"+preheaderdata[3];
5392 prefsize = int(preheaderdata[5], 16);
5393 prefcompression = preheaderdata[12];
5394 prefcsize = int(preheaderdata[13], 16);
5395 prefseeknextfile = preheaderdata[25];
5396 prefextrasize = int(preheaderdata[26], 16);
5397 prefextrafields = int(preheaderdata[27], 16);
5398 extrafieldslist = [];
5399 extrastart = 28;
5400 extraend = extrastart + prefextrafields;
5401 extrafieldslist = [];
5402 if(extrastart<extraend):
5403 extrafieldslist.append(preheaderdata[extrastart]);
5404 extrastart = extrastart + 1;
5405 prefcs = preheaderdata[-2].lower();
5406 prenewfcs = preheaderdata[-1].lower();
5407 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
5408 if(prefcs!=prenewfcs and not skipchecksum):
5409 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
5410 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
5411 return False;
5412 valid_archive = False;
5413 invalid_archive = True;
5414 prefhend = catfp.tell() - 1;
5415 prefcontentstart = catfp.tell();
5416 prefcontents = "";
5417 pyhascontents = False;
5418 if(prefsize>0):
5419 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
5420 prefcontents = catfp.read(prefsize);
5421 else:
5422 prefcontents = catfp.read(prefcsize);
5423 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
5424 pyhascontents = True;
5425 if(prefccs!=prenewfccs and not skipchecksum):
5426 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
5427 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
5428 return False;
5429 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
5430 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
5431 if(abs(fseeknextasnum)==0):
5432 pass;
5433 catfp.seek(fseeknextasnum, 1);
5434 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
5435 fseeknextasnum = int(prefseeknextfile);
5436 if(abs(fseeknextasnum)==0):
5437 pass;
5438 catfp.seek(fseeknextasnum, 1);
5439 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
5440 fseeknextasnum = int(prefseeknextfile);
5441 if(abs(fseeknextasnum)==0):
5442 pass;
5443 catfp.seek(fseeknextasnum, 0);
5444 else:
5445 return False;
5446 il = il + 1;
5447 fileidnum = seekstart;
5448 realidnum = 0;
5449 while(fileidnum<seekend):
5450 catfhstart = catfp.tell();
5451 if(formatspecs['new_style']):
5452 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
5453 else:
5454 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
5455 if(len(catheaderdata)==0):
5456 break;
5457 catfheadsize = int(catheaderdata[0], 16);
5458 catfnumfields = int(catheaderdata[1], 16);
5459 catftype = int(catheaderdata[2], 16);
5460 if(re.findall(r"^[.|/]", catheaderdata[3])):
5461 catfname = catheaderdata[3];
5462 else:
5463 catfname = "./"+catheaderdata[3];
5464 catfbasedir = os.path.dirname(catfname);
5465 catflinkname = catheaderdata[4];
5466 catfsize = int(catheaderdata[5], 16);
5467 catfatime = int(catheaderdata[6], 16);
5468 catfmtime = int(catheaderdata[7], 16);
5469 catfctime = int(catheaderdata[8], 16);
5470 catfbtime = int(catheaderdata[9], 16);
5471 catfmode = int(catheaderdata[10], 16);
5472 catfchmode = stat.S_IMODE(catfmode);
5473 catftypemod = stat.S_IFMT(catfmode);
5474 catfwinattributes = int(catheaderdata[11], 16);
5475 catfcompression = catheaderdata[12];
5476 catfcsize = int(catheaderdata[13], 16);
5477 catfuid = int(catheaderdata[14], 16);
5478 catfuname = catheaderdata[15];
5479 catfgid = int(catheaderdata[16], 16);
5480 catfgname = catheaderdata[17];
5481 catfid = int(catheaderdata[18], 16);
5482 catfinode = int(catheaderdata[19], 16);
5483 catflinkcount = int(catheaderdata[20], 16);
5484 catfdev_minor = int(catheaderdata[21], 16);
5485 catfdev_major = int(catheaderdata[22], 16);
5486 catfrdev_minor = int(catheaderdata[23], 16);
5487 catfrdev_major = int(catheaderdata[24], 16);
5488 catfseeknextfile = catheaderdata[25];
5489 catfextrasize = int(catheaderdata[26], 16);
5490 catfextrafields = int(catheaderdata[27], 16);
5491 extrafieldslist = [];
5492 extrastart = 28;
5493 extraend = extrastart + catfextrafields;
5494 extrafieldslist = [];
5495 if(extrastart<extraend):
5496 extrafieldslist.append(catheaderdata[extrastart]);
5497 extrastart = extrastart + 1;
5498 catfcs = catheaderdata[-2].lower();
5499 catfccs = catheaderdata[-1].lower();
5500 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5501 if(catfcs!=catnewfcs and not skipchecksum):
5502 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
5503 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5504 return False;
5505 catfhend = catfp.tell() - 1;
5506 catfcontentstart = catfp.tell();
5507 catfcontents = BytesIO();
5508 pyhascontents = False;
5509 if(catfsize>0 and not listonly):
5510 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5511 catfcontents.write(catfp.read(catfsize));
5512 else:
5513 catfcontents.write(catfp.read(catfcsize));
5514 catfcontents.seek(0, 0);
5515 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5516 pyhascontents = True;
5517 if(catfccs!=catnewfccs and skipchecksum):
5518 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5519 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5520 return False;
5521 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5522 pass;
5523 else:
5524 catfcontents.seek(0, 0);
5525 if(uncompress):
5526 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5527 catfcontents.seek(0, 0);
5528 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5529 if(catfsize>0 and listonly):
5530 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5531 catfp.seek(catfsize, 1);
5532 else:
5533 catfp.seek(catfcsize, 1);
5534 pyhascontents = False;
5535 catfcontentend = catfp.tell();
5536 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5537 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5538 if(abs(fseeknextasnum)==0):
5539 pass;
5540 catfp.seek(fseeknextasnum, 1);
5541 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5542 fseeknextasnum = int(catfseeknextfile);
5543 if(abs(fseeknextasnum)==0):
5544 pass;
5545 catfp.seek(fseeknextasnum, 1);
5546 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5547 fseeknextasnum = int(catfseeknextfile);
5548 if(abs(fseeknextasnum)==0):
5549 pass;
5550 catfp.seek(fseeknextasnum, 0);
5551 else:
5552 return False;
5553 catfcontents.seek(0, 0);
5554 if(not contentasfile):
5555 catfcontents = catfcontents.read();
5556 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontentasfile': contentasfile, 'fcontents': catfcontents});
5557 fileidnum = fileidnum + 1;
5558 realidnum = realidnum + 1;
5559 if(returnfp):
5560 catlist.update({'catfp': catfp});
5561 else:
5562 catfp.close();
5563 return catlist;
5564
5565 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5566
5567 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5568 formatspecs = FormatSpecsListToDict(formatspecs);
5569 catfp = BytesIO(catstr);
5570 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, contentasfile, True, skipchecksum, formatspecs, returnfp);
5571 return listcatfiles;
5572
5573 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5574
5575 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5576 formatspecs = FormatSpecsListToDict(formatspecs);
5577 catfp = BytesIO();
5578 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5579 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, contentasfile, True, skipchecksum, formatspecs, returnfp);
5580 return listcatfiles;
5581
5582 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5583 formatspecs = FormatSpecsListToDict(formatspecs);
5584 catfp = BytesIO();
5585 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5586 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, contentasfile, True, skipchecksum, formatspecs, returnfp);
5587 return listcatfiles;
5588
5589 if(not rarfile_support):
5590 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5591 return False;
5592
5593 if(rarfile_support):
5594 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5595 formatspecs = FormatSpecsListToDict(formatspecs);
5596 catfp = BytesIO();
5597 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5598 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, contentasfile, True, skipchecksum, formatspecs, returnfp);
5599 return listcatfiles;
5600
5601 if(not py7zr_support):
5602 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5603 return False;
5604
5605 if(py7zr_support):
5606 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5607 formatspecs = FormatSpecsListToDict(formatspecs);
5608 catfp = BytesIO();
5609 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5610 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, contentasfile, True, skipchecksum, formatspecs, returnfp);
5611 return listcatfiles;
5612
5613 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, contentasfile=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5614 formatspecs = FormatSpecsListToDict(formatspecs);
5615 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5616 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
5617 return TarFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5618 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
5619 return ZipFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5620 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
5621 return RarFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5622 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
5623 return SevenZipFileToArray(infile, seekstart, seekend, listonly, contentasfile, skipchecksum, formatspecs, returnfp);
5624 elif(checkcompressfile=="catfile"):
5625 return ArchiveFileToArray(infile, seekstart, seekend, listonly, contentasfile, True, skipchecksum, formatspecs, returnfp);
5626 else:
5627 return False;
5628 return False;
5629
5630 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5631 formatspecs = FormatSpecsListToDict(formatspecs);
5632 catver = formatspecs['format_ver'];
5633 fileheaderver = str(int(catver.replace(".", "")));
5634 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5635 advancedlist = formatspecs['use_advanced_list'];
5636 altinode = formatspecs['use_alt_inode'];
5637 infilelist = [];
5638 if(infiles=="-"):
5639 for line in sys.stdin:
5640 infilelist.append(line.strip());
5641 infilelist = list(filter(None, infilelist));
5642 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5643 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5644 return False;
5645 with UncompressFile(infiles, formatspecs, "r") as finfile:
5646 for line in finfile:
5647 infilelist.append(line.strip());
5648 infilelist = list(filter(None, infilelist));
5649 else:
5650 if(isinstance(infiles, (list, tuple, ))):
5651 infilelist = list(filter(None, infiles));
5652 elif(isinstance(infiles, (str, ))):
5653 infilelist = list(filter(None, [infiles]));
5654 if(advancedlist):
5655 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5656 else:
5657 GetDirList = ListDir(infilelist, followlink, False);
5658 if(not GetDirList):
5659 return False;
5660 curinode = 0;
5661 curfid = 0;
5662 inodelist = [];
5663 inodetofile = {};
5664 filetoinode = {};
5665 inodetocatinode = {};
5666 fileidnum = 0;
5667 fnumfiles = int(len(GetDirList));
5668 catver = formatspecs['format_ver'];
5669 fileheaderver = str(int(catver.replace(".", "")));
5670 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5671 fnumfileshex = format(int(fnumfiles), 'x').lower();
5672 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5673 catversion = re.findall(r"([\d]+)", fileheader);
5674 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5675 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5676 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5677 fheadtell = len(fileheader);
5678 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5679 for curfname in GetDirList:
5680 catfhstart = fheadtell;
5681 if(re.findall(r"^[.|/]", curfname)):
5682 fname = curfname;
5683 else:
5684 fname = "./"+curfname;
5685 if(verbose):
5686 VerbosePrintOut(fname);
5687 if(not followlink or followlink is None):
5688 fstatinfo = os.lstat(fname);
5689 else:
5690 fstatinfo = os.stat(fname);
5691 fpremode = fstatinfo.st_mode;
5692 finode = fstatinfo.st_ino;
5693 flinkcount = fstatinfo.st_nlink;
5694 ftype = 0;
5695 if(stat.S_ISREG(fpremode)):
5696 ftype = 0;
5697 elif(stat.S_ISLNK(fpremode)):
5698 ftype = 2;
5699 elif(stat.S_ISCHR(fpremode)):
5700 ftype = 3;
5701 elif(stat.S_ISBLK(fpremode)):
5702 ftype = 4;
5703 elif(stat.S_ISDIR(fpremode)):
5704 ftype = 5;
5705 elif(stat.S_ISFIFO(fpremode)):
5706 ftype = 6;
5707 elif(stat.S_ISSOCK(fpremode)):
5708 ftype = 8;
5709 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5710 ftype = 9;
5711 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5712 ftype = 10;
5713 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5714 ftype = 11;
5715 else:
5716 ftype = 0;
5717 flinkname = "";
5718 fbasedir = os.path.dirname(fname);
5719 fcurfid = curfid;
5720 if(not followlink and finode!=0):
5721 if(ftype!=1):
5722 if(finode in inodelist):
5723 ftype = 1;
5724 flinkname = inodetofile[finode];
5725 if(altinode):
5726 fcurinode = finode;
5727 else:
5728 fcurinode = inodetocatinode[finode];
5729 if(finode not in inodelist):
5730 inodelist.append(finode);
5731 inodetofile.update({finode: fname});
5732 inodetocatinode.update({finode: curinode});
5733 if(altinode):
5734 fcurinode = finode;
5735 else:
5736 fcurinode = curinode;
5737 curinode = curinode + 1;
5738 else:
5739 fcurinode = curinode;
5740 curinode = curinode + 1;
5741 curfid = curfid + 1;
5742 if(ftype==2):
5743 flinkname = os.readlink(fname);
5744 fdev = fstatinfo.st_dev;
5745 getfdev = GetDevMajorMinor(fdev);
5746 fdev_minor = getfdev[0];
5747 fdev_major = getfdev[1];
5748 frdev = fstatinfo.st_dev;
5749 if(hasattr(fstatinfo, "st_rdev")):
5750 frdev = fstatinfo.st_rdev;
5751 else:
5752 frdev = fstatinfo.st_dev;
5753 getfrdev = GetDevMajorMinor(frdev);
5754 frdev_minor = getfrdev[0];
5755 frdev_major = getfrdev[1];
5756 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5757 fsize = "0";
5758 if(ftype==0 or ftype==7):
5759 fsize = fstatinfo.st_size;
5760 fatime = fstatinfo.st_atime;
5761 fmtime = fstatinfo.st_mtime;
5762 fctime = fstatinfo.st_ctime;
5763 if(hasattr(fstatinfo, "st_birthtime")):
5764 fbtime = fstatinfo.st_birthtime;
5765 else:
5766 fbtime = fstatinfo.st_ctime;
5767 fmode = fstatinfo.st_mode;
5768 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5769 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5770 fuid = fstatinfo.st_uid;
5771 fgid = fstatinfo.st_gid;
5772 funame = "";
5773 try:
5774 import pwd;
5775 try:
5776 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5777 funame = userinfo.pw_name;
5778 except KeyError:
5779 funame = "";
5780 except ImportError:
5781 funame = "";
5782 fgname = "";
5783 try:
5784 import grp;
5785 try:
5786 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5787 fgname = groupinfo.gr_name;
5788 except KeyError:
5789 fgname = "";
5790 except ImportError:
5791 fgname = "";
5792 fdev_minor = fdev_minor;
5793 fdev_major = fdev_major;
5794 frdev_minor = frdev_minor;
5795 frdev_major = frdev_major;
5796 flinkcount = flinkcount;
5797 if(hasattr(fstatinfo, "st_file_attributes")):
5798 fwinattributes = fstatinfo.st_file_attributes;
5799 else:
5800 fwinattributes = 0;
5801 fcompression = "";
5802 fcsize = 0;
5803 fcontents = BytesIO();
5804 if(ftype==0 or ftype==7):
5805 with open(fname, "rb") as fpc:
5806 shutil.copyfileobj(fpc, fcontents);
5807 if(followlink and (ftype==1 or ftype==2)):
5808 flstatinfo = os.stat(flinkname);
5809 with open(flinkname, "rb") as fpc:
5810 shutil.copyfileobj(fpc, fcontents);
5811 fcontents.seek(0, 0);
5812 ftypehex = format(ftype, 'x').lower();
5813 extrafields = len(extradata);
5814 extrafieldslist = extradata;
5815 catfextrafields = extrafields;
5816 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5817 if(len(extradata)>0):
5818 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5819 extrasizelen = len(extrasizestr);
5820 extrasizelenhex = format(extrasizelen, 'x').lower();
5821 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5822 catoutlen = len(catoutlist) + len(extradata) + 3;
5823 catoutlenhex = format(catoutlen, 'x').lower();
5824 catoutlist.insert(0, catoutlenhex);
5825 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5826 catheaderdata = catoutlist;
5827 if(len(extradata)>0):
5828 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5829 if(fsize==0):
5830 checksumlist = [checksumtype, "none"];
5831 else:
5832 checksumlist = [checksumtype, checksumtype];
5833 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5834 catfnumfields = catoutlen;
5835 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5836 fcontents.seek(0, 0);
5837 if(fsize==0):
5838 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5839 else:
5840 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5841 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5842 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5843 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5844 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5845 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5846 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5847 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5848 catfcontentstart = fheadtell;
5849 fheadtell += len(catfileoutstr) + 1;
5850 catfcontentend = fheadtell - 1;
5851 catfhend = catfcontentend;
5852 fcontents.seek(0, 0);
5853 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5854 pyhascontents = False;
5855 if(int(fsize)>0 and not listonly):
5856 pyhascontents = True;
5857 if(int(fsize)>0 and listonly):
5858 fcontents = BytesIO();
5859 pyhascontents = False;
5860 fcontents.seek(0, 0);
5861 if(not contentasfile):
5862 fcontents = fcontents.read();
5863 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontentasfile': contentasfile, 'fcontents': fcontents});
5864 fileidnum = fileidnum + 1;
5865 return catlist;
5866
5867 def TarFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5868 formatspecs = FormatSpecsListToDict(formatspecs);
5869 curinode = 0;
5870 curfid = 0;
5871 inodelist = [];
5872 inodetofile = {};
5873 filetoinode = {};
5874 inodetocatinode = {};
5875 fileidnum = 0;
5876 if(infile=="-"):
5877 infile = BytesIO();
5878 if(hasattr(sys.stdin, "buffer")):
5879 shutil.copyfileobj(sys.stdin.buffer, infile);
5880 else:
5881 shutil.copyfileobj(sys.stdin, infile);
5882 infile.seek(0, 0);
5883 if(not infile):
5884 return False;
5885 infile.seek(0, 0);
5886 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5887 infile = download_file_from_internet_file(infile);
5888 infile.seek(0, 0);
5889 if(not infile):
5890 return False;
5891 infile.seek(0, 0);
5892 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5893 return False;
5894 elif(os.path.exists(infile) and os.path.isfile(infile)):
5895 try:
5896 if(not tarfile.TarFileCheck(infile)):
5897 return False;
5898 except AttributeError:
5899 if(not TarFileCheck(infile)):
5900 return False;
5901 try:
5902 if(hasattr(infile, "read") or hasattr(infile, "write")):
5903 tarfp = tarfile.open(fileobj=infile, mode="r");
5904 else:
5905 tarfp = tarfile.open(infile, "r");
5906 except FileNotFoundError:
5907 return False;
5908 fnumfiles = int(len(tarfp.getmembers()));
5909 catver = formatspecs['format_ver'];
5910 fileheaderver = str(int(catver.replace(".", "")));
5911 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5912 fnumfileshex = format(int(fnumfiles), 'x').lower();
5913 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5914 catversion = re.findall(r"([\d]+)", fileheader);
5915 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5916 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5917 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5918 fheadtell = len(fileheader);
5919 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5920 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5921 catfhstart = fheadtell;
5922 if(re.findall(r"^[.|/]", member.name)):
5923 fname = member.name;
5924 else:
5925 fname = "./"+member.name;
5926 if(verbose):
5927 VerbosePrintOut(fname);
5928 fpremode = member.mode;
5929 ffullmode = member.mode;
5930 flinkcount = 0;
5931 ftype = 0;
5932 if(member.isreg()):
5933 ffullmode = member.mode + stat.S_IFREG;
5934 ftype = 0;
5935 elif(member.isdev()):
5936 ffullmode = member.mode;
5937 ftype = 7;
5938 elif(member.islnk()):
5939 ffullmode = member.mode + stat.S_IFREG;
5940 ftype = 1;
5941 elif(member.issym()):
5942 ffullmode = member.mode + stat.S_IFLNK;
5943 ftype = 2;
5944 elif(member.ischr()):
5945 ffullmode = member.mode + stat.S_IFCHR;
5946 ftype = 3;
5947 elif(member.isblk()):
5948 ffullmode = member.mode + stat.S_IFBLK;
5949 ftype = 4;
5950 elif(member.isdir()):
5951 ffullmode = member.mode + stat.S_IFDIR;
5952 ftype = 5;
5953 elif(member.isfifo()):
5954 ffullmode = member.mode + stat.S_IFIFO;
5955 ftype = 6;
5956 elif(member.issparse()):
5957 ffullmode = member.mode;
5958 ftype = 12;
5959 else:
5960 ffullmode = member.mode;
5961 ftype = 0;
5962 flinkname = "";
5963 fbasedir = os.path.dirname(fname);
5964 fcurfid = curfid;
5965 fcurinode = curfid;
5966 finode = fcurinode;
5967 curfid = curfid + 1;
5968 if(ftype==2):
5969 flinkname = member.linkname;
5970 fdev_minor = member.devminor;
5971 fdev_major = member.devmajor;
5972 frdev_minor = member.devminor;
5973 frdev_major = member.devmajor;
5974 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5975 fsize = "0";
5976 elif(ftype==0 or ftype==7):
5977 fsize = member.size;
5978 else:
5979 fsize = member.size;
5980 fatime = member.mtime;
5981 fmtime = member.mtime;
5982 fctime = member.mtime;
5983 fbtime = member.mtime;
5984 fmode = ffullmode;
5985 fchmode = stat.S_IMODE(ffullmode);
5986 ftypemod = stat.S_IFMT(ffullmode);
5987 fuid = member.uid;
5988 fgid = member.gid;
5989 funame = member.uname;
5990 fgname = member.gname;
5991 flinkcount = flinkcount;
5992 fwinattributes = int(0);
5993 fcompression = "";
5994 fcsize = 0;
5995 fcontents = BytesIO();
5996 if(ftype==0 or ftype==7):
5997 with tarfp.extractfile(member) as fpc:
5998 shutil.copyfileobj(fpc, fcontents);
5999 fcontents.seek(0, 0);
6000 ftypehex = format(ftype, 'x').lower();
6001 extrafields = len(extradata);
6002 extrafieldslist = extradata;
6003 catfextrafields = extrafields;
6004 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6005 if(len(extradata)>0):
6006 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6007 extrasizelen = len(extrasizestr);
6008 extrasizelenhex = format(extrasizelen, 'x').lower();
6009 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6010 catoutlen = len(catoutlist) + len(extradata) + 3;
6011 catoutlenhex = format(catoutlen, 'x').lower();
6012 catoutlist.insert(0, catoutlenhex);
6013 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6014 catheaderdata = catoutlist;
6015 if(len(extradata)>0):
6016 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6017 if(fsize==0):
6018 checksumlist = [checksumtype, "none"];
6019 else:
6020 checksumlist = [checksumtype, checksumtype];
6021 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6022 catfnumfields = catoutlen;
6023 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6024 fcontents.seek(0, 0);
6025 if(fsize==0):
6026 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6027 else:
6028 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6029 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6030 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6031 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6032 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6033 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6034 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6035 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6036 catfcontentstart = fheadtell;
6037 fheadtell += len(catfileoutstr) + 1;
6038 catfcontentend = fheadtell - 1;
6039 catfhend = catfcontentend;
6040 fcontents.seek(0, 0);
6041 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6042 pyhascontents = False;
6043 if(int(fsize)>0 and not listonly):
6044 pyhascontents = True;
6045 if(int(fsize)>0 and listonly):
6046 fcontents = BytesIO();
6047 pyhascontents = False;
6048 fcontents.seek(0, 0);
6049 if(not contentasfile):
6050 fcontents = fcontents.read();
6051 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontentasfile': contentasfile, 'fcontents': fcontents});
6052 fileidnum = fileidnum + 1;
6053 return catlist;
6054
6055 def ZipFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6056 formatspecs = FormatSpecsListToDict(formatspecs);
6057 curinode = 0;
6058 curfid = 0;
6059 inodelist = [];
6060 inodetofile = {};
6061 filetoinode = {};
6062 inodetocatinode = {};
6063 fileidnum = 0;
6064 if(infile=="-"):
6065 infile = BytesIO();
6066 if(hasattr(sys.stdin, "buffer")):
6067 shutil.copyfileobj(sys.stdin.buffer, infile);
6068 else:
6069 shutil.copyfileobj(sys.stdin, infile);
6070 infile.seek(0, 0);
6071 if(not infile):
6072 return False;
6073 infile.seek(0, 0);
6074 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6075 infile = download_file_from_internet_file(infile);
6076 infile.seek(0, 0);
6077 if(not infile):
6078 return False;
6079 infile.seek(0, 0);
6080 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6081 return False;
6082 if(not zipfile.is_zipfile(infile)):
6083 return False;
6084 try:
6085 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6086 except FileNotFoundError:
6087 return False;
6088 ziptest = zipfp.testzip();
6089 if(ziptest):
6090 VerbosePrintOut("Bad file found!");
6091 fnumfiles = int(len(zipfp.infolist()));
6092 catver = formatspecs['format_ver'];
6093 fileheaderver = str(int(catver.replace(".", "")));
6094 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6095 catversion = re.findall(r"([\d]+)", fileheader);
6096 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6097 fnumfileshex = format(int(fnumfiles), 'x').lower();
6098 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6099 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6100 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6101 fheadtell = len(fileheader);
6102 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6103 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6104 catfhstart = fheadtell;
6105 if(re.findall(r"^[.|/]", member.filename)):
6106 fname = member.filename;
6107 else:
6108 fname = "./"+member.filename;
6109 zipinfo = zipfp.getinfo(member.filename);
6110 if(verbose):
6111 VerbosePrintOut(fname);
6112 if(not member.is_dir()):
6113 fpremode = stat.S_IFREG + 438;
6114 elif(member.is_dir()):
6115 fpremode = stat.S_IFDIR + 511;
6116 flinkcount = 0;
6117 ftype = 0;
6118 if(not member.is_dir()):
6119 ftype = 0;
6120 elif(member.is_dir()):
6121 ftype = 5;
6122 flinkname = "";
6123 fbasedir = os.path.dirname(fname);
6124 fcurfid = curfid;
6125 fcurinode = curfid;
6126 finode = fcurinode;
6127 curfid = curfid + 1;
6128 fdev_minor = 0;
6129 fdev_major = 0;
6130 frdev_minor = 0;
6131 frdev_major = 0;
6132 if(ftype==5):
6133 fsize = "0";
6134 elif(ftype==0):
6135 fsize = member.file_size;
6136 else:
6137 fsize = member.file_size;
6138 fatime = time.mktime(member.date_time + (0, 0, -1));
6139 fmtime = time.mktime(member.date_time + (0, 0, -1));
6140 fctime = time.mktime(member.date_time + (0, 0, -1));
6141 fbtime = time.mktime(member.date_time + (0, 0, -1));
6142 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6143 fwinattributes = int(zipinfo.external_attr);
6144 if(not member.is_dir()):
6145 fmode = int(stat.S_IFREG + 438);
6146 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6147 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6148 elif(member.is_dir()):
6149 fmode = int(stat.S_IFDIR + 511);
6150 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6151 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6152 elif(zipinfo.create_system==3):
6153 fwinattributes = int(0);
6154 try:
6155 fmode = int(zipinfo.external_attr);
6156 fchmode = stat.S_IMODE(fmode);
6157 ftypemod = stat.S_IFMT(fmode);
6158 except OverflowError:
6159 fmode = int(zipinfo.external_attr >> 16);
6160 fchmode = stat.S_IMODE(fmode);
6161 ftypemod = stat.S_IFMT(fmode);
6162 else:
6163 fwinattributes = int(0);
6164 if(not member.is_dir()):
6165 fmode = int(stat.S_IFREG + 438);
6166 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6167 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6168 elif(member.is_dir()):
6169 fmode = int(stat.S_IFDIR + 511);
6170 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6171 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6172 fcompression = "";
6173 fcsize = 0;
6174 try:
6175 fuid = os.getuid();
6176 except AttributeError:
6177 fuid = 0;
6178 except KeyError:
6179 fuid = 0;
6180 try:
6181 fgid = os.getgid();
6182 except AttributeError:
6183 fgid = 0;
6184 except KeyError:
6185 fgid = 0;
6186 try:
6187 import pwd;
6188 try:
6189 userinfo = pwd.getpwuid(os.getuid());
6190 funame = userinfo.pw_name;
6191 except KeyError:
6192 funame = "";
6193 except AttributeError:
6194 funame = "";
6195 except ImportError:
6196 funame = "";
6197 fgname = "";
6198 try:
6199 import grp;
6200 try:
6201 groupinfo = grp.getgrgid(os.getgid());
6202 fgname = groupinfo.gr_name;
6203 except KeyError:
6204 fgname = "";
6205 except AttributeError:
6206 fgname = "";
6207 except ImportError:
6208 fgname = "";
6209 fcontents = BytesIO();
6210 if(ftype==0):
6211 fcontents.write(zipfp.read(member.filename));
6212 fcontents.seek(0, 0);
6213 ftypehex = format(ftype, 'x').lower();
6214 extrafields = len(extradata);
6215 extrafieldslist = extradata;
6216 catfextrafields = extrafields;
6217 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6218 if(len(extradata)>0):
6219 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6220 extrasizelen = len(extrasizestr);
6221 extrasizelenhex = format(extrasizelen, 'x').lower();
6222 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6223 catoutlen = len(catoutlist) + len(extradata) + 3;
6224 catoutlenhex = format(catoutlen, 'x').lower();
6225 catoutlist.insert(0, catoutlenhex);
6226 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6227 catheaderdata = catoutlist;
6228 if(len(extradata)>0):
6229 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6230 if(fsize==0):
6231 checksumlist = [checksumtype, "none"];
6232 else:
6233 checksumlist = [checksumtype, checksumtype];
6234 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6235 catfnumfields = catoutlen;
6236 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6237 fcontents.seek(0, 0);
6238 if(fsize==0):
6239 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6240 else:
6241 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6242 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6243 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6244 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6245 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6246 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6247 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6248 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6249 catfcontentstart = fheadtell;
6250 fheadtell += len(catfileoutstr) + 1;
6251 catfcontentend = fheadtell - 1;
6252 catfhend = catfcontentend;
6253 fcontents.seek(0, 0);
6254 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6255 pyhascontents = False;
6256 if(int(fsize)>0 and not listonly):
6257 pyhascontents = True;
6258 if(int(fsize)>0 and listonly):
6259 fcontents = BytesIO();
6260 pyhascontents = False;
6261 fcontents.seek(0, 0);
6262 if(not contentasfile):
6263 fcontents = fcontents.read();
6264 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontentasfile': contentasfile, 'fcontents': fcontents});
6265 fileidnum = fileidnum + 1;
6266 return catlist;
6267
6268 if(not rarfile_support):
6269 def RarFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6270 return False;
6271
6272 if(rarfile_support):
6273 def RarFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6274 formatspecs = FormatSpecsListToDict(formatspecs);
6275 curinode = 0;
6276 curfid = 0;
6277 inodelist = [];
6278 inodetofile = {};
6279 filetoinode = {};
6280 inodetocatinode = {};
6281 fileidnum = 0;
6282 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
6283 return False;
6284 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6285 return False;
6286 rarfp = rarfile.RarFile(infile, "r");
6287 rartest = rarfp.testrar();
6288 if(rartest):
6289 VerbosePrintOut("Bad file found!");
6290 fnumfiles = int(len(rarfp.infolist()));
6291 catver = formatspecs['format_ver'];
6292 fileheaderver = str(int(catver.replace(".", "")));
6293 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6294 catversion = re.findall(r"([\d]+)", fileheader);
6295 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6296 fnumfileshex = format(int(fnumfiles), 'x').lower();
6297 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6298 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6299 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6300 fheadtell = len(fileheader);
6301 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6302 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6303 catfhstart = fheadtell;
6304 is_unix = False;
6305 is_windows = False;
6306 if(member.host_os==rarfile.RAR_OS_UNIX):
6307 is_windows = False;
6308 try:
6309 member.external_attr
6310 is_unix = True;
6311 except AttributeError:
6312 is_unix = False;
6313 elif(member.host_os==rarfile.RAR_OS_WIN32):
6314 is_unix = False;
6315 try:
6316 member.external_attr
6317 is_windows = True;
6318 except AttributeError:
6319 is_windows = False;
6320 else:
6321 is_unix = False;
6322 is_windows = False;
6323 if(re.findall(r"^[.|/]", member.filename)):
6324 fname = member.filename;
6325 else:
6326 fname = "./"+member.filename;
6327 rarinfo = rarfp.getinfo(member.filename);
6328 if(verbose):
6329 VerbosePrintOut(fname);
6330 if(is_unix and member.external_attr !=0):
6331 fpremode = int(member.external_attr);
6332 elif(member.is_file()):
6333 fpremode = stat.S_IFREG + 438;
6334 elif(member.is_symlink()):
6335 fpremode = stat.S_IFLNK + 438;
6336 elif(member.is_dir()):
6337 fpremode = stat.S_IFDIR + 511;
6338 if(is_windows and member.external_attr !=0):
6339 fwinattributes = int(member.external_attr);
6340 else:
6341 fwinattributes = int(0);
6342 fcompression = "";
6343 fcsize = 0;
6344 flinkcount = 0;
6345 ftype = 0;
6346 if(member.is_file()):
6347 ftype = 0;
6348 elif(member.is_symlink()):
6349 ftype = 2;
6350 elif(member.is_dir()):
6351 ftype = 5;
6352 flinkname = "";
6353 if(ftype==2):
6354 flinkname = rarfp.read(member.filename).decode("UTF-8");
6355 fbasedir = os.path.dirname(fname);
6356 fcurfid = curfid;
6357 fcurinode = curfid;
6358 finode = fcurinode;
6359 curfid = curfid + 1;
6360 fdev_minor = 0;
6361 fdev_major = 0;
6362 frdev_minor = 0;
6363 frdev_major = 0;
6364 if(ftype==5):
6365 fsize = "0";
6366 if(ftype==0):
6367 fsize = member.file_size;
6368 try:
6369 if(member.atime):
6370 fatime = int(member.atime.timestamp());
6371 else:
6372 fatime = int(member.mtime.timestamp());
6373 except AttributeError:
6374 fatime = int(member.mtime.timestamp());
6375 fmtime = int(member.mtime.timestamp());
6376 try:
6377 if(member.ctime):
6378 fctime = int(member.ctime.timestamp());
6379 else:
6380 fctime = int(member.mtime.timestamp());
6381 except AttributeError:
6382 fctime = int(member.mtime.timestamp());
6383 fbtime = int(member.mtime.timestamp());
6384 if(is_unix and member.external_attr !=0):
6385 fmode = int(member.external_attr);
6386 fchmode = int(stat.S_IMODE(member.external_attr));
6387 ftypemod = int(stat.S_IFMT(member.external_attr));
6388 elif(member.is_file()):
6389 fmode = int(stat.S_IFREG + 438)
6390 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6391 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6392 elif(member.is_symlink()):
6393 fmode = int(stat.S_IFLNK + 438)
6394 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6395 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6396 elif(member.is_dir()):
6397 fmode = int(stat.S_IFDIR + 511)
6398 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6399 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6400 try:
6401 fuid = os.getuid();
6402 except AttributeError:
6403 fuid = 0;
6404 except KeyError:
6405 fuid = 0;
6406 try:
6407 fgid = os.getgid();
6408 except AttributeError:
6409 fgid = 0;
6410 except KeyError:
6411 fgid = 0;
6412 try:
6413 import pwd;
6414 try:
6415 userinfo = pwd.getpwuid(os.getuid());
6416 funame = userinfo.pw_name;
6417 except KeyError:
6418 funame = "";
6419 except AttributeError:
6420 funame = "";
6421 except ImportError:
6422 funame = "";
6423 fgname = "";
6424 try:
6425 import grp;
6426 try:
6427 groupinfo = grp.getgrgid(os.getgid());
6428 fgname = groupinfo.gr_name;
6429 except KeyError:
6430 fgname = "";
6431 except AttributeError:
6432 fgname = "";
6433 except ImportError:
6434 fgname = "";
6435 fcontents = BytesIO();
6436 if(ftype==0):
6437 fcontents.write(rarfp.read(member.filename));
6438 fcontents.seek(0, 0);
6439 ftypehex = format(ftype, 'x').lower();
6440 extrafields = len(extradata);
6441 extrafieldslist = extradata;
6442 catfextrafields = extrafields;
6443 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6444 if(len(extradata)>0):
6445 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6446 extrasizelen = len(extrasizestr);
6447 extrasizelenhex = format(extrasizelen, 'x').lower();
6448 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6449 catoutlen = len(catoutlist) + len(extradata) + 3;
6450 catoutlenhex = format(catoutlen, 'x').lower();
6451 catoutlist.insert(0, catoutlenhex);
6452 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6453 if(len(extradata)>0):
6454 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6455 if(fsize==0):
6456 checksumlist = [checksumtype, "none"];
6457 else:
6458 checksumlist = [checksumtype, checksumtype];
6459 ccatfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6460 catfnumfields = 24 + catfextrafields;
6461 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6462 fcontents.seek(0, 0);
6463 if(fsize==0):
6464 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6465 else:
6466 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6467 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6468 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6469 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6470 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6471 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6472 catheaderdata = catoutlist;
6473 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6474 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6475 catfcontentstart = fheadtell;
6476 fheadtell += len(catfileoutstr) + 1;
6477 catfcontentend = fheadtell - 1;
6478 catfhend = catfcontentend;
6479 fcontents.seek(0, 0);
6480 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6481 pyhascontents = False;
6482 if(int(fsize)>0 and not listonly):
6483 pyhascontents = True;
6484 if(int(fsize)>0 and listonly):
6485 fcontents = BytesIO();
6486 pyhascontents = False;
6487 fcontents.seek(0, 0);
6488 if(not contentasfile):
6489 fcontents = fcontents.read();
6490 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontentasfile': contentasfile, 'fcontents': fcontents});
6491 fileidnum = fileidnum + 1;
6492 return catlist;
6493
6494 if(not py7zr_support):
6495 def SevenZipFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6496 return False;
6497
6498 if(py7zr_support):
6499 def SevenZipFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6500 formatspecs = FormatSpecsListToDict(formatspecs);
6501 curinode = 0;
6502 curfid = 0;
6503 inodelist = [];
6504 inodetofile = {};
6505 filetoinode = {};
6506 inodetocatinode = {};
6507 fileidnum = 0;
6508 szpfp = py7zr.SevenZipFile(infile, mode="r");
6509 file_content = szpfp.readall();
6510 #sztest = szpfp.testzip();
6511 sztestalt = szpfp.test();
6512 if(sztestalt):
6513 VerbosePrintOut("Bad file found!");
6514 numfiles = int(len(szpfp.list()));
6515 catver = formatspecs['format_ver'];
6516 fileheaderver = str(int(catver.replace(".", "")));
6517 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
6518 catversion = re.findall(r"([\d]+)", fileheader);
6519 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
6520 fnumfileshex = format(int(fnumfiles), 'x').lower();
6521 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
6522 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
6523 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
6524 fheadtell = len(fileheader);
6525 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
6526 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6527 catfhstart = fheadtell;
6528 if(re.findall(r"^[.|/]", member.filename)):
6529 fname = member.filename;
6530 else:
6531 fname = "./"+member.filename;
6532 if(not member.is_directory):
6533 fpremode = int(stat.S_IFREG + 438);
6534 elif(member.is_directory):
6535 fpremode = int(stat.S_IFDIR + 511);
6536 fwinattributes = int(0);
6537 fcompression = "";
6538 fcsize = 0;
6539 flinkcount = 0;
6540 ftype = 0;
6541 if(member.is_directory):
6542 ftype = 5;
6543 else:
6544 ftype = 0;
6545 flinkname = "";
6546 fbasedir = os.path.dirname(fname);
6547 fcurfid = curfid;
6548 fcurinode = curfid;
6549 finode = fcurinode;
6550 curfid = curfid + 1;
6551 fdev_minor = 0;
6552 fdev_major = 0;
6553 frdev_minor = 0;
6554 frdev_major = 0;
6555 if(ftype==5):
6556 fsize = "0";
6557 fatime = int(member.creationtime.timestamp());
6558 fmtime = int(member.creationtime.timestamp());
6559 fctime = int(member.creationtime.timestamp());
6560 fbtime = int(member.creationtime.timestamp());
6561 if(member.is_directory):
6562 fmode = int(stat.S_IFDIR + 511)
6563 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6564 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6565 else:
6566 fmode = int(stat.S_IFLNK + 438)
6567 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6568 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6569 try:
6570 fuid = os.getuid();
6571 except AttributeError:
6572 fuid = 0;
6573 except KeyError:
6574 fuid = 0;
6575 try:
6576 fgid = os.getgid();
6577 except AttributeError:
6578 fgid = 0;
6579 except KeyError:
6580 fgid = 0;
6581 try:
6582 import pwd;
6583 try:
6584 userinfo = pwd.getpwuid(os.getuid());
6585 funame = userinfo.pw_name;
6586 except KeyError:
6587 funame = "";
6588 except AttributeError:
6589 funame = "";
6590 except ImportError:
6591 funame = "";
6592 fgname = "";
6593 try:
6594 import grp;
6595 try:
6596 groupinfo = grp.getgrgid(os.getgid());
6597 fgname = groupinfo.gr_name;
6598 except KeyError:
6599 fgname = "";
6600 except AttributeError:
6601 fgname = "";
6602 except ImportError:
6603 fgname = "";
6604 fcontents = BytesIO();
6605 if(ftype==0):
6606 fcontents.write(file_content[member.filename].read());
6607 fsize = format(fcontents.tell(), 'x').lower();
6608 fileop.close();
6609 fcontents.seek(0, 0);
6610 ftypehex = format(ftype, 'x').lower();
6611 extrafields = len(extradata);
6612 extrafieldslist = extradata;
6613 catfextrafields = extrafields;
6614 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6615 if(len(extradata)>0):
6616 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6617 extrasizelen = len(extrasizestr);
6618 extrasizelenhex = format(extrasizelen, 'x').lower();
6619 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6620 catoutlen = len(catoutlist) + len(extradata) + 3;
6621 catoutlenhex = format(catoutlen, 'x').lower();
6622 catoutlist.insert(0, catoutlenhex);
6623 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6624 catheaderdata = catoutlist;
6625 if(len(extradata)>0):
6626 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6627 if(fsize==0):
6628 checksumlist = [checksumtype, "none"];
6629 else:
6630 checksumlist = [checksumtype, checksumtype];
6631 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
6632 catfnumfields = 24 + catfextrafields;
6633 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6634 fcontents.seek(0, 0);
6635 if(fsize==0):
6636 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
6637 else:
6638 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6639 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6640 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6641 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6642 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6643 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6644 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6645 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6646 catfcontentstart = fheadtell;
6647 fheadtell += len(catfileoutstr) + 1;
6648 catfcontentend = fheadtell - 1;
6649 catfhend = catfcontentend;
6650 fcontents.seek(0, 0);
6651 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6652 pyhascontents = False;
6653 if(int(fsize)>0 and not listonly):
6654 pyhascontents = True;
6655 if(int(fsize)>0 and listonly):
6656 fcontents = BytesIO();
6657 pyhascontents = False;
6658 fcontents.seek(0, 0);
6659 if(not contentasfile):
6660 fcontents = fcontents.read();
6661 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontentasfile': contentasfile, 'fcontents': fcontents});
6662 fileidnum = fileidnum + 1;
6663 return catlist;
6664
6665 def InFileToArrayAlt(infile, listonly=False, contentasfile=True, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6666 formatspecs = FormatSpecsListToDict(formatspecs);
6667 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6668 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
6669 return TarFileToArrayAlt(infile, listonly, contentasfile, checksumtype, extradata, formatspecs, verbose);
6670 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
6671 return ZipFileToArrayAlt(infile, listonly, contentasfile, checksumtype, extradata, formatspecs, verbose);
6672 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
6673 return RarFileToArrayAlt(infile, listonly, contentasfile, checksumtype, extradata, formatspecs, verbose);
6674 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
6675 return SevenZipFileToArrayAlt(infile, listonly, contentasfile, checksumtype, extradata, formatspecs, verbose);
6676 elif(checkcompressfile=="catfile"):
6677 return ArchiveFileToArray(infile, 0, 0, listonly, contentasfile, True, False, formatspecs, False);
6678 else:
6679 return False;
6680 return False;
6681
6682 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6683 formatspecs = FormatSpecsListToDict(formatspecs);
6684 outarray = BytesIO();
6685 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6686 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6687 return listcatfiles;
6688
6689 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6690 formatspecs = FormatSpecsListToDict(formatspecs);
6691 if(isinstance(inarray, dict)):
6692 listcatfiles = inarray;
6693 else:
6694 return False;
6695 if(not listcatfiles):
6696 return False;
6697 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6698 if(returnfp):
6699 catarray.update({'catfp': listcatfiles['catfp']});
6700 lenlist = len(listcatfiles['ffilelist']);
6701 lcfi = 0;
6702 lcfx = int(listcatfiles['fnumfiles']);
6703 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6704 lcfx = int(lenlist);
6705 else:
6706 lcfx = int(listcatfiles['fnumfiles']);
6707 while(lcfi < lcfx):
6708 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6709 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6710 catarray['filetoid'].update(filetoidarray);
6711 catarray['idtofile'].update(idtofilearray);
6712 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6713 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6714 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6715 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6716 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6717 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6718 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6719 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6720 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6721 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6722 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6723 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6724 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6725 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6726 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6727 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6728 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6729 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6730 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6731 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6732 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6733 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6734 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6735 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6736 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6737 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6738 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6739 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6740 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6741 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6742 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6743 lcfi = lcfi + 1;
6744 return catarray;
6745
6746 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6747
6748 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6749 formatspecs = FormatSpecsListToDict(formatspecs);
6750 if(isinstance(infile, dict)):
6751 listcatfiles = infile;
6752 else:
6753 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6754 infile = RemoveWindowsPath(infile);
6755 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6756 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6757 outfile = RemoveWindowsPath(outfile);
6758 checksumtype = checksumtype.lower();
6759 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6760 checksumtype="crc32";
6761 if(checksumtype=="none"):
6762 checksumtype = "";
6763 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6764 compression = "auto";
6765 if(compression not in compressionlist and compression is None):
6766 compression = "auto";
6767 if(verbose):
6768 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6769 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6770 if(os.path.exists(outfile)):
6771 try:
6772 os.unlink(outfile);
6773 except OSError as e:
6774 pass;
6775 if(not listcatfiles):
6776 return False;
6777 if(outfile=="-"):
6778 verbose = False;
6779 catfp = BytesIO();
6780 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6781 catfp = outfile;
6782 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6783 catfp = BytesIO();
6784 else:
6785 fbasename = os.path.splitext(outfile)[0];
6786 fextname = os.path.splitext(outfile)[1];
6787 if(not compresswholefile and fextname in outextlistwd):
6788 compresswholefile = True;
6789 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6790 catver = formatspecs['format_ver'];
6791 fileheaderver = str(int(catver.replace(".", "")));
6792 lenlist = len(listcatfiles['ffilelist']);
6793 fnumfiles = int(listcatfiles['fnumfiles']);
6794 if(lenlist>fnumfiles or lenlist<fnumfiles):
6795 fnumfiles = lenlist;
6796 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6797 lenlist = len(listcatfiles['ffilelist']);
6798 fnumfiles = int(listcatfiles['fnumfiles']);
6799 lcfi = 0;
6800 lcfx = int(listcatfiles['fnumfiles']);
6801 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6802 lcfx = int(lenlist);
6803 else:
6804 lcfx = int(listcatfiles['fnumfiles']);
6805 curinode = 0;
6806 curfid = 0;
6807 inodelist = [];
6808 inodetofile = {};
6809 filetoinode = {};
6810 reallcfi = 0;
6811 while(lcfi < lcfx):
6812 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6813 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6814 else:
6815 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6816 if(verbose):
6817 VerbosePrintOut(fname);
6818 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6819 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6820 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6821 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6822 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6823 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6824 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6825 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6826 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6827 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6828 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6829 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6830 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6831 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6832 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6833 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6834 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6835 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6836 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6837 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6838 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6839 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6840 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6841 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6842 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6843 if(not followlink and len(extradata)<0):
6844 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6845 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6846 if(not listcatfiles['ffilelist'][reallcfi]['fcontentasfile']):
6847 fcontents = BytesIO(fcontents);
6848 fcompression = "";
6849 fcsize = format(int(0), 'x').lower();
6850 if(not compresswholefile):
6851 fcontents.seek(0, 2);
6852 ucfsize = fcontents.tell();
6853 fcontents.seek(0, 0);
6854 if(compression=="auto"):
6855 ilsize = len(compressionlistalt);
6856 ilmin = 0;
6857 ilcsize = [];
6858 while(ilmin < ilsize):
6859 cfcontents = BytesIO();
6860 shutil.copyfileobj(fcontents, cfcontents);
6861 fcontents.seek(0, 0);
6862 cfcontents.seek(0, 0);
6863 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6864 if(cfcontents):
6865 cfcontents.seek(0, 2);
6866 ilcsize.append(cfcontents.tell());
6867 cfcontents.close();
6868 else:
6869 try:
6870 ilcsize.append(sys.maxint);
6871 except AttributeError:
6872 ilcsize.append(sys.maxsize);
6873 ilmin = ilmin + 1;
6874 ilcmin = ilcsize.index(min(ilcsize));
6875 compression = compressionlistalt[ilcmin];
6876 fcontents.seek(0, 0);
6877 cfcontents = BytesIO();
6878 shutil.copyfileobj(fcontents, cfcontents);
6879 cfcontents.seek(0, 0);
6880 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6881 cfcontents.seek(0, 2);
6882 cfsize = cfcontents.tell();
6883 if(ucfsize > cfsize):
6884 fcsize = format(int(cfsize), 'x').lower();
6885 fcompression = compression;
6886 fcontents.close();
6887 fcontents = cfcontents;
6888 if(followlink):
6889 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6890 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6891 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6892 flinkinfo = listcatfiles['ffilelist'][flinkid];
6893 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6894 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6895 flinkname = flinkinfo['flinkname'];
6896 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6897 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6898 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6899 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6900 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6901 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6902 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6903 funame = flinkinfo['funame'];
6904 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6905 fgname = flinkinfo['fgname'];
6906 finode = format(int(flinkinfo['finode']), 'x').lower();
6907 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6908 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6909 fcompression = flinkinfo['fcompression'];
6910 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6911 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6912 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6913 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6914 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6915 fseeknextfile = flinkinfo['fseeknextfile'];
6916 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6917 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6918 if(len(extradata)<0):
6919 extradata = flinkinfo['fextralist'];
6920 fcontents = flinkinfo['fcontents'];
6921 if(not flinkinfo['fcontentasfile']):
6922 fcontents = BytesIO(fcontents);
6923 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6924 else:
6925 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6926 fcurfid = format(curfid, 'x').lower();
6927 if(not followlink and finode!=0):
6928 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6929 fcurinode = format(int(curinode), 'x').lower();
6930 inodetofile.update({curinode: fname});
6931 filetoinode.update({fname: curinode});
6932 curinode = curinode + 1;
6933 else:
6934 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6935 else:
6936 fcurinode = format(int(curinode), 'x').lower();
6937 curinode = curinode + 1;
6938 curfid = curfid + 1;
6939 if(fcompression=="none"):
6940 fcompression = "";
6941 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6942 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6943 fcontents.close();
6944 lcfi = lcfi + 1;
6945 reallcfi = reallcfi + 1;
6946 if(lcfx>0):
6947 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6948 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6949 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6950 try:
6951 catfp.flush();
6952 os.fsync(catfp.fileno());
6953 except io.UnsupportedOperation:
6954 pass;
6955 except AttributeError:
6956 pass;
6957 except OSError as e:
6958 pass;
6959 if(outfile=="-"):
6960 catfp.seek(0, 0);
6961 if(hasattr(sys.stdout, "buffer")):
6962 shutil.copyfileobj(catfp, sys.stdout.buffer);
6963 else:
6964 shutil.copyfileobj(catfp, sys.stdout);
6965 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6966 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6967 catfp.seek(0, 0);
6968 upload_file_to_internet_file(catfp, outfile);
6969 if(returnfp):
6970 catfp.seek(0, 0);
6971 return catfp;
6972 else:
6973 catfp.close();
6974 return True;
6975
6976 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6977
6978 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6979 formatspecs = FormatSpecsListToDict(formatspecs);
6980 catfp = BytesIO(catstr);
6981 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6982 return listcatfiles;
6983
6984 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6985
6986 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6987 formatspecs = FormatSpecsListToDict(formatspecs);
6988 outarray = BytesIO();
6989 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6990 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6991 return listcatfiles;
6992
6993 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6994
6995 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6996 formatspecs = FormatSpecsListToDict(formatspecs);
6997 if(outdir is not None):
6998 outdir = RemoveWindowsPath(outdir);
6999 if(verbose):
7000 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7001 if(isinstance(infile, dict)):
7002 listcatfiles = infile;
7003 else:
7004 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
7005 infile = RemoveWindowsPath(infile);
7006 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
7007 if(not listcatfiles):
7008 return False;
7009 lenlist = len(listcatfiles['ffilelist']);
7010 fnumfiles = int(listcatfiles['fnumfiles']);
7011 lcfi = 0;
7012 lcfx = int(listcatfiles['fnumfiles']);
7013 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
7014 lcfx = int(lenlist);
7015 else:
7016 lcfx = int(listcatfiles['fnumfiles']);
7017 while(lcfi<lcfx):
7018 funame = "";
7019 try:
7020 import pwd;
7021 try:
7022 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
7023 funame = userinfo.pw_name;
7024 except KeyError:
7025 funame = "";
7026 except ImportError:
7027 funame = "";
7028 fgname = "";
7029 try:
7030 import grp;
7031 try:
7032 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
7033 fgname = groupinfo.gr_name;
7034 except KeyError:
7035 fgname = "";
7036 except ImportError:
7037 fgname = "";
7038 if(verbose):
7039 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7040 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
7041 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
7042 if(not listcatfiles['ffilelist'][lcfi]['fcontentasfile']):
7043 listcatfiles['ffilelist'][lcfi]['fcontents'] = BytesIO(listcatfiles['ffilelist'][lcfi]['fcontents']);
7044 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
7045 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
7046 try:
7047 fpc.flush();
7048 os.fsync(fpc.fileno());
7049 except io.UnsupportedOperation:
7050 pass;
7051 except AttributeError:
7052 pass;
7053 except OSError as e:
7054 pass;
7055 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
7056 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
7057 if(preservepermissions):
7058 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7059 if(preservetime):
7060 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
7061 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
7062 if(followlink):
7063 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
7064 flinkid = prelistcatfiles['filetoid'][getflinkpath];
7065 flinkinfo = listcatfiles['ffilelist'][flinkid];
7066 funame = "";
7067 try:
7068 import pwd;
7069 try:
7070 userinfo = pwd.getpwuid(flinkinfo['fuid']);
7071 funame = userinfo.pw_name;
7072 except KeyError:
7073 funame = "";
7074 except ImportError:
7075 funame = "";
7076 fgname = "";
7077 try:
7078 import grp;
7079 try:
7080 groupinfo = grp.getgrgid(flinkinfo['fgid']);
7081 fgname = groupinfo.gr_name;
7082 except KeyError:
7083 fgname = "";
7084 except ImportError:
7085 fgname = "";
7086 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
7087 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
7088 if(not flinkinfo['fcontentasfile']):
7089 flinkinfo['fcontents'] = BytesIO(flinkinfo['fcontents']);
7090 flinkinfo['fcontents'].seek(0, 0);
7091 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
7092 try:
7093 fpc.flush();
7094 os.fsync(fpc.fileno());
7095 except io.UnsupportedOperation:
7096 pass;
7097 except AttributeError:
7098 pass;
7099 except OSError as e:
7100 pass;
7101 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7102 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7103 if(preservepermissions):
7104 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7105 if(preservetime):
7106 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7107 if(flinkinfo['ftype']==1):
7108 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7109 if(flinkinfo['ftype']==2):
7110 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7111 if(flinkinfo['ftype']==5):
7112 if(preservepermissions):
7113 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7114 else:
7115 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7116 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7117 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7118 if(preservepermissions):
7119 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7120 if(preservetime):
7121 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7122 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
7123 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7124 else:
7125 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7126 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
7127 if(followlink):
7128 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
7129 flinkid = prelistcatfiles['filetoid'][getflinkpath];
7130 flinkinfo = listcatfiles['ffilelist'][flinkid];
7131 funame = "";
7132 try:
7133 import pwd;
7134 try:
7135 userinfo = pwd.getpwuid(flinkinfo['fuid']);
7136 funame = userinfo.pw_name;
7137 except KeyError:
7138 funame = "";
7139 except ImportError:
7140 funame = "";
7141 fgname = "";
7142 try:
7143 import grp;
7144 try:
7145 groupinfo = grp.getgrgid(flinkinfo['fgid']);
7146 fgname = groupinfo.gr_name;
7147 except KeyError:
7148 fgname = "";
7149 except ImportError:
7150 fgname = "";
7151 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
7152 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
7153 if(not flinkinfo['fcontentasfile']):
7154 flinkinfo['fcontents'] = BytesIO(flinkinfo['fcontents']);
7155 flinkinfo['fcontents'].seek(0, 0);
7156 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
7157 try:
7158 fpc.flush();
7159 os.fsync(fpc.fileno());
7160 except io.UnsupportedOperation:
7161 pass;
7162 except AttributeError:
7163 pass;
7164 except OSError as e:
7165 pass;
7166 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7167 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7168 if(preservepermissions):
7169 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7170 if(preservetime):
7171 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7172 if(flinkinfo['ftype']==1):
7173 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7174 if(flinkinfo['ftype']==2):
7175 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7176 if(flinkinfo['ftype']==5):
7177 if(preservepermissions):
7178 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7179 else:
7180 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7181 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
7182 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
7183 if(preservepermissions):
7184 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7185 if(preservetime):
7186 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
7187 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
7188 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
7189 else:
7190 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7191 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
7192 if(preservepermissions):
7193 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7194 else:
7195 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
7196 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
7197 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
7198 if(preservepermissions):
7199 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7200 if(preservetime):
7201 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
7202 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
7203 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
7204 lcfi = lcfi + 1;
7205 if(returnfp):
7206 return listcatfiles['ffilelist']['catfp'];
7207 else:
7208 return True;
7209
7210 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
7211
7212 if(hasattr(shutil, "register_unpack_format")):
7213 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
7214 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
7215 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
7216
7217 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7218 formatspecs = FormatSpecsListToDict(formatspecs);
7219 catfp = BytesIO(catstr);
7220 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7221 return listcatfiles;
7222
7223 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
7224
7225 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7226 formatspecs = FormatSpecsListToDict(formatspecs);
7227 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7228 if(isinstance(infile, dict)):
7229 listcatfiles = infile;
7230 else:
7231 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
7232 infile = RemoveWindowsPath(infile);
7233 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
7234 if(not listcatfiles):
7235 return False;
7236 lenlist = len(listcatfiles['ffilelist']);
7237 fnumfiles = int(listcatfiles['fnumfiles']);
7238 lcfi = 0;
7239 lcfx = int(listcatfiles['fnumfiles']);
7240 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
7241 lcfx = int(lenlist);
7242 else:
7243 lcfx = int(listcatfiles['fnumfiles']);
7244 returnval = {};
7245 while(lcfi<lcfx):
7246 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
7247 if(not verbose):
7248 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
7249 if(verbose):
7250 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7251 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
7252 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
7253 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
7254 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
7255 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
7256 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
7257 if(len(fuprint)<=0):
7258 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
7259 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
7260 if(len(fgprint)<=0):
7261 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
7262 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7263 lcfi = lcfi + 1;
7264 if(returnfp):
7265 return listcatfiles['catfp'];
7266 else:
7267 return True;
7268
7269 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
7270
7271 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7272 formatspecs = FormatSpecsListToDict(formatspecs);
7273 catfp = BytesIO(catstr);
7274 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7275 return listcatfiles;
7276
7277 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
7278
7279 def TarFileListFiles(infile, verbose=False, returnfp=False):
7280 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7281 if(infile=="-"):
7282 infile = BytesIO();
7283 if(hasattr(sys.stdin, "buffer")):
7284 shutil.copyfileobj(sys.stdin.buffer, infile);
7285 else:
7286 shutil.copyfileobj(sys.stdin, infile);
7287 infile.seek(0, 0);
7288 if(not infile):
7289 return False;
7290 infile.seek(0, 0);
7291 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7292 infile = download_file_from_internet_file(infile);
7293 infile.seek(0, 0);
7294 if(not infile):
7295 return False;
7296 infile.seek(0, 0);
7297 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7298 return False;
7299 elif(os.path.exists(infile) and os.path.isfile(infile)):
7300 try:
7301 if(not tarfile.TarFileCheck(infile)):
7302 return False;
7303 except AttributeError:
7304 if(not TarFileCheck(infile)):
7305 return False;
7306 try:
7307 if(hasattr(infile, "read") or hasattr(infile, "write")):
7308 tarfp = tarfile.open(fileobj=infile, mode="r");
7309 else:
7310 tarfp = tarfile.open(infile, "r");
7311 except FileNotFoundError:
7312 return False;
7313 lcfi = 0
7314 returnval = {};
7315 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
7316 returnval.update({lcfi: member.name});
7317 fpremode = member.mode;
7318 ffullmode = member.mode;
7319 flinkcount = 0;
7320 ftype = 0;
7321 if(member.isreg()):
7322 ffullmode = member.mode + stat.S_IFREG;
7323 ftype = 0;
7324 elif(member.isdev()):
7325 ffullmode = member.mode;
7326 ftype = 7;
7327 elif(member.islnk()):
7328 ffullmode = member.mode + stat.S_IFREG;
7329 ftype = 1;
7330 elif(member.issym()):
7331 ffullmode = member.mode + stat.S_IFLNK;
7332 ftype = 2;
7333 elif(member.ischr()):
7334 ffullmode = member.mode + stat.S_IFCHR;
7335 ftype = 3;
7336 elif(member.isblk()):
7337 ffullmode = member.mode + stat.S_IFBLK;
7338 ftype = 4;
7339 elif(member.isdir()):
7340 ffullmode = member.mode + stat.S_IFDIR;
7341 ftype = 5;
7342 elif(member.isfifo()):
7343 ffullmode = member.mode + stat.S_IFIFO;
7344 ftype = 6;
7345 elif(member.issparse()):
7346 ffullmode = member.mode;
7347 ftype = 12;
7348 if(not verbose):
7349 VerbosePrintOut(member.name);
7350 elif(verbose):
7351 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7352 printfname = member.name;
7353 if(member.islnk()):
7354 printfname = member.name + " link to " + member.linkname;
7355 elif(member.issym()):
7356 printfname = member.name + " -> " + member.linkname;
7357 fuprint = member.uname;
7358 if(len(fuprint)<=0):
7359 fuprint = member.uid;
7360 fgprint = member.gname;
7361 if(len(fgprint)<=0):
7362 fgprint = member.gid;
7363 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7364 lcfi = lcfi + 1;
7365 if(returnfp):
7366 return listcatfiles['catfp'];
7367 else:
7368 return True;
7369
7370 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7371 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7372 if(infile=="-"):
7373 infile = BytesIO();
7374 if(hasattr(sys.stdin, "buffer")):
7375 shutil.copyfileobj(sys.stdin.buffer, infile);
7376 else:
7377 shutil.copyfileobj(sys.stdin, infile);
7378 infile.seek(0, 0);
7379 if(not infile):
7380 return False;
7381 infile.seek(0, 0);
7382 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7383 infile = download_file_from_internet_file(infile);
7384 infile.seek(0, 0);
7385 if(not infile):
7386 return False;
7387 infile.seek(0, 0);
7388 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7389 return False;
7390 if(not zipfile.is_zipfile(infile)):
7391 return False;
7392 try:
7393 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7394 except FileNotFoundError:
7395 print(6);return False;
7396 lcfi = 0;
7397 returnval = {};
7398 ziptest = zipfp.testzip();
7399 if(ziptest):
7400 VerbosePrintOut("Bad file found!");
7401 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7402 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7403 fwinattributes = int(zipinfo.external_attr);
7404 if(not member.is_dir()):
7405 fmode = int(stat.S_IFREG + 438);
7406 fchmode = int(stat.S_IMODE(fmode));
7407 ftypemod = int(stat.S_IFMT(fmode));
7408 elif(member.is_dir()):
7409 fmode = int(stat.S_IFDIR + 511);
7410 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7411 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7412 elif(zipinfo.create_system==3):
7413 fwinattributes =int(0);
7414 try:
7415 fmode = int(zipinfo.external_attr);
7416 fchmode = stat.S_IMODE(fmode);
7417 ftypemod = stat.S_IFMT(fmode);
7418 except OverflowError:
7419 fmode = int(zipinfo.external_attr >> 16);
7420 fchmode = stat.S_IMODE(fmode);
7421 ftypemod = stat.S_IFMT(fmode);
7422 else:
7423 fwinattributes = int(0);
7424 if(not member.is_dir()):
7425 fmode = int(stat.S_IFREG + 438);
7426 fchmode = int(stat.S_IMODE(fmode));
7427 ftypemod = int(stat.S_IFMT(fmode));
7428 elif(member.is_dir()):
7429 fmode = int(stat.S_IFDIR + 511);
7430 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7431 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7432 returnval.update({lcfi: member.filename});
7433 if(not verbose):
7434 VerbosePrintOut(member.filename);
7435 if(verbose):
7436 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7437 permissionstr = "";
7438 for fmodval in str(oct(fmode))[-3:]:
7439 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7440 if(not member.is_dir()):
7441 ftype = 0;
7442 permissionstr = "-" + permissionstr;
7443 elif(member.is_dir()):
7444 ftype = 5;
7445 permissionstr = "d" + permissionstr;
7446 printfname = member.filename;
7447 try:
7448 fuid = int(os.getuid());
7449 except AttributeError:
7450 fuid = int(0);
7451 except KeyError:
7452 fuid = int(0);
7453 try:
7454 fgid = int(os.getgid());
7455 except AttributeError:
7456 fgid = int(0);
7457 except KeyError:
7458 fgid = int(0);
7459 try:
7460 import pwd;
7461 try:
7462 userinfo = pwd.getpwuid(os.getuid());
7463 funame = userinfo.pw_name;
7464 except KeyError:
7465 funame = "";
7466 except AttributeError:
7467 funame = "";
7468 except ImportError:
7469 funame = "";
7470 fgname = "";
7471 try:
7472 import grp;
7473 try:
7474 groupinfo = grp.getgrgid(os.getgid());
7475 fgname = groupinfo.gr_name;
7476 except KeyError:
7477 fgname = "";
7478 except AttributeError:
7479 fgname = "";
7480 except ImportError:
7481 fgname = "";
7482 fuprint = funame;
7483 if(len(fuprint)<=0):
7484 fuprint = str(fuid);
7485 fgprint = fgname;
7486 if(len(fgprint)<=0):
7487 fgprint = str(fgid);
7488 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7489 lcfi = lcfi + 1;
7490 if(returnfp):
7491 return listcatfiles['catfp'];
7492 else:
7493 return True;
7494
7495 if(not rarfile_support):
7496 def RarFileListFiles(infile, verbose=False, returnfp=False):
7497 return False;
7498
7499 if(rarfile_support):
7500 def RarFileListFiles(infile, verbose=False, returnfp=False):
7501 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7502 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7503 return False;
7504 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7505 return False;
7506 lcfi = 0;
7507 returnval = {};
7508 rarfp = rarfile.RarFile(infile, "r");
7509 rartest = rarfp.testrar();
7510 if(rartest):
7511 VerbosePrintOut("Bad file found!");
7512 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7513 is_unix = False;
7514 is_windows = False;
7515 if(member.host_os==rarfile.RAR_OS_UNIX):
7516 is_windows = False;
7517 try:
7518 member.external_attr
7519 is_unix = True;
7520 except AttributeError:
7521 is_unix = False;
7522 elif(member.host_os==rarfile.RAR_OS_WIN32):
7523 is_unix = False;
7524 try:
7525 member.external_attr
7526 is_windows = True;
7527 except AttributeError:
7528 is_windows = False;
7529 else:
7530 is_unix = False;
7531 is_windows = False;
7532 if(is_unix and member.external_attr !=0):
7533 fpremode = int(member.external_attr);
7534 elif(member.is_file()):
7535 fpremode = int(stat.S_IFREG + 438);
7536 elif(member.is_symlink()):
7537 fpremode = int(stat.S_IFLNK + 438);
7538 elif(member.is_dir()):
7539 fpremode = int(stat.S_IFDIR + 511);
7540 if(is_windows and member.external_attr !=0):
7541 fwinattributes = int(member.external_attr);
7542 else:
7543 fwinattributes = int(0);
7544 if(is_unix and member.external_attr !=0):
7545 fmode = int(member.external_attr);
7546 fchmode = int(stat.S_IMODE(member.external_attr));
7547 ftypemod = int(stat.S_IFMT(member.external_attr));
7548 elif(member.is_file()):
7549 fmode = int(stat.S_IFREG + 438);
7550 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7551 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7552 elif(member.is_symlink()):
7553 fmode = int(stat.S_IFLNK + 438);
7554 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7555 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7556 elif(member.is_dir()):
7557 fmode = int(stat.S_IFDIR + 511);
7558 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7559 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7560 returnval.update({lcfi: member.filename});
7561 if(not verbose):
7562 VerbosePrintOut(member.filename);
7563 if(verbose):
7564 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7565 permissionstr = "";
7566 for fmodval in str(oct(fmode))[-3:]:
7567 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7568 if(member.is_file()):
7569 ftype = 0;
7570 permissionstr = "-" + permissionstr;
7571 printfname = member.filename;
7572 elif(member.is_symlink()):
7573 ftype = 2;
7574 permissionstr = "l" + permissionstr;
7575 printfname = member.name + " -> " + member.read().decode("UTF-8");
7576 elif(member.is_dir()):
7577 ftype = 5;
7578 permissionstr = "d" + permissionstr;
7579 printfname = member.filename;
7580 try:
7581 fuid = int(os.getuid());
7582 except AttributeError:
7583 fuid = int(0);
7584 except KeyError:
7585 fuid = int(0);
7586 try:
7587 fgid = int(os.getgid());
7588 except AttributeError:
7589 fgid = int(0);
7590 except KeyError:
7591 fgid = int(0);
7592 try:
7593 import pwd;
7594 try:
7595 userinfo = pwd.getpwuid(os.getuid());
7596 funame = userinfo.pw_name;
7597 except KeyError:
7598 funame = "";
7599 except AttributeError:
7600 funame = "";
7601 except ImportError:
7602 funame = "";
7603 fgname = "";
7604 try:
7605 import grp;
7606 try:
7607 groupinfo = grp.getgrgid(os.getgid());
7608 fgname = groupinfo.gr_name;
7609 except KeyError:
7610 fgname = "";
7611 except AttributeError:
7612 fgname = "";
7613 except ImportError:
7614 fgname = "";
7615 fuprint = funame;
7616 if(len(fuprint)<=0):
7617 fuprint = str(fuid);
7618 fgprint = fgname;
7619 if(len(fgprint)<=0):
7620 fgprint = str(fgid);
7621 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7622 lcfi = lcfi + 1;
7623 if(returnfp):
7624 return listcatfiles['catfp'];
7625 else:
7626 return True;
7627
7628 if(not py7zr_support):
7629 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7630 return False;
7631
7632 if(py7zr_support):
7633 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7634 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7635 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7636 return False;
7637 lcfi = 0;
7638 returnval = {};
7639 szpfp = py7zr.SevenZipFile(infile, mode="r");
7640 file_content = szpfp.readall();
7641 #sztest = szpfp.testzip();
7642 sztestalt = szpfp.test();
7643 if(sztestalt):
7644 VerbosePrintOut("Bad file found!");
7645 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7646 if(re.findall(r"^[.|/]", member.filename)):
7647 fname = member.filename;
7648 else:
7649 fname = "./"+member.filename;
7650 if(not member.is_directory):
7651 fpremode = int(stat.S_IFREG + 438);
7652 elif(member.is_directory):
7653 fpremode = int(stat.S_IFDIR + 511);
7654 fwinattributes = int(0);
7655 if(member.is_directory):
7656 fmode = int(stat.S_IFDIR + 511);
7657 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7658 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7659 else:
7660 fmode = int(stat.S_IFLNK + 438);
7661 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7662 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7663 returnval.update({lcfi: member.filename});
7664 if(not verbose):
7665 VerbosePrintOut(member.filename);
7666 if(verbose):
7667 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7668 permissionstr = "";
7669 for fmodval in str(oct(fmode))[-3:]:
7670 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7671 fsize = int("0");
7672 if(not member.is_directory):
7673 ftype = 0;
7674 permissionstr = "-" + permissionstr;
7675 printfname = member.filename;
7676 elif(member.is_directory):
7677 ftype = 5;
7678 permissionstr = "d" + permissionstr;
7679 printfname = member.filename;
7680 if(ftype==0):
7681 fsize = len(file_content[member.filename].read());
7682 file_content[member.filename].close();
7683 try:
7684 fuid = int(os.getuid());
7685 except AttributeError:
7686 fuid = int(0);
7687 except KeyError:
7688 fuid = int(0);
7689 try:
7690 fgid = int(os.getgid());
7691 except AttributeError:
7692 fgid = int(0);
7693 except KeyError:
7694 fgid = int(0);
7695 try:
7696 import pwd;
7697 try:
7698 userinfo = pwd.getpwuid(os.getuid());
7699 funame = userinfo.pw_name;
7700 except KeyError:
7701 funame = "";
7702 except AttributeError:
7703 funame = "";
7704 except ImportError:
7705 funame = "";
7706 fgname = "";
7707 try:
7708 import grp;
7709 try:
7710 groupinfo = grp.getgrgid(os.getgid());
7711 fgname = groupinfo.gr_name;
7712 except KeyError:
7713 fgname = "";
7714 except AttributeError:
7715 fgname = "";
7716 except ImportError:
7717 fgname = "";
7718 fuprint = funame;
7719 if(len(fuprint)<=0):
7720 fuprint = str(fuid);
7721 fgprint = fgname;
7722 if(len(fgprint)<=0):
7723 fgprint = str(fgid);
7724 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7725 lcfi = lcfi + 1;
7726 if(returnfp):
7727 return listcatfiles['catfp'];
7728 else:
7729 return True;
7730
7731 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7732 formatspecs = FormatSpecsListToDict(formatspecs);
7733 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7734 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7735 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
7736 return TarFileListFiles(infile, verbose, returnfp);
7737 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
7738 return ZipFileListFiles(infile, verbose, returnfp);
7739 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
7740 return RarFileListFiles(infile, verbose, returnfp);
7741 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
7742 return SevenZipFileListFiles(infile, verbose, returnfp);
7743 elif(checkcompressfile=="catfile"):
7744 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7745 else:
7746 return False;
7747 return False;
7748
7749 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7750 formatspecs = FormatSpecsListToDict(formatspecs);
7751 outarray = BytesIO();
7752 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7753 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7754 return listcatfiles;
7755
7756 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, contentasfile=True, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7757 formatspecs = FormatSpecsListToDict(formatspecs);
7758 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, contentasfile, checksumtype, formatspecs, verbose);
7759 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7760 return listcatfiles;
7761
7762 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7763 formatspecs = FormatSpecsListToDict(formatspecs);
7764 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, True, checksumtype, extradata, formatspecs, False);
7765 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7766 return listcatfiles;
7767
7768 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7769
7770 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7771 formatspecs = FormatSpecsListToDict(formatspecs);
7772 outarray = TarFileToArrayAlt(infile, False, True, checksumtype, extradata, formatspecs, False);
7773 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7774 return listcatfiles;
7775
7776 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7777
7778 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7779 formatspecs = FormatSpecsListToDict(formatspecs);
7780 outarray = ZipFileToArrayAlt(infile, False, True, checksumtype, extradata, formatspecs, False);
7781 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7782 return listcatfiles;
7783
7784 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7785
7786 if(not rarfile_support):
7787 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7788 return False;
7789
7790 if(rarfile_support):
7791 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7792 formatspecs = FormatSpecsListToDict(formatspecs);
7793 outarray = RarFileToArrayAlt(infile, False, True, checksumtype, extradata, formatspecs, False);
7794 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7795 return listcatfiles;
7796
7797 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7798
7799 if(not py7zr_support):
7800 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7801 return False;
7802
7803 if(py7zr_support):
7804 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7805 formatspecs = FormatSpecsListToDict(formatspecs);
7806 outarray = SevenZipFileToArrayAlt(infile, False, True, checksumtype, extradata, formatspecs, False);
7807 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7808 return listcatfiles;
7809
7810 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7811
7812 def download_file_from_ftp_file(url):
7813 urlparts = urlparse(url);
7814 file_name = os.path.basename(urlparts.path);
7815 file_dir = os.path.dirname(urlparts.path);
7816 if(urlparts.username is not None):
7817 ftp_username = urlparts.username;
7818 else:
7819 ftp_username = "anonymous";
7820 if(urlparts.password is not None):
7821 ftp_password = urlparts.password;
7822 elif(urlparts.password is None and urlparts.username=="anonymous"):
7823 ftp_password = "anonymous";
7824 else:
7825 ftp_password = "";
7826 if(urlparts.scheme=="ftp"):
7827 ftp = FTP();
7828 elif(urlparts.scheme=="ftps" and ftpssl):
7829 ftp = FTP_TLS();
7830 else:
7831 return False;
7832 if(urlparts.scheme=="sftp"):
7833 if(__use_pysftp__):
7834 return download_file_from_pysftp_file(url);
7835 else:
7836 return download_file_from_sftp_file(url);
7837 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7838 return download_file_from_http_file(url);
7839 ftp_port = urlparts.port;
7840 if(urlparts.port is None):
7841 ftp_port = 21;
7842 try:
7843 ftp.connect(urlparts.hostname, ftp_port);
7844 except socket.gaierror:
7845 log.info("Error With URL "+url);
7846 return False;
7847 except socket.timeout:
7848 log.info("Error With URL "+url);
7849 return False;
7850 ftp.login(urlparts.username, urlparts.password);
7851 if(urlparts.scheme=="ftps"):
7852 ftp.prot_p();
7853 ftpfile = BytesIO();
7854 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7855 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7856 ftp.close();
7857 ftpfile.seek(0, 0);
7858 return ftpfile;
7859
7860 def download_file_from_ftp_string(url):
7861 ftpfile = download_file_from_ftp_file(url);
7862 return ftpfile.read();
7863
7864 def upload_file_to_ftp_file(ftpfile, url):
7865 urlparts = urlparse(url);
7866 file_name = os.path.basename(urlparts.path);
7867 file_dir = os.path.dirname(urlparts.path);
7868 if(urlparts.username is not None):
7869 ftp_username = urlparts.username;
7870 else:
7871 ftp_username = "anonymous";
7872 if(urlparts.password is not None):
7873 ftp_password = urlparts.password;
7874 elif(urlparts.password is None and urlparts.username=="anonymous"):
7875 ftp_password = "anonymous";
7876 else:
7877 ftp_password = "";
7878 if(urlparts.scheme=="ftp"):
7879 ftp = FTP();
7880 elif(urlparts.scheme=="ftps" and ftpssl):
7881 ftp = FTP_TLS();
7882 else:
7883 return False;
7884 if(urlparts.scheme=="sftp"):
7885 if(__use_pysftp__):
7886 return upload_file_to_pysftp_file(url);
7887 else:
7888 return upload_file_to_sftp_file(url);
7889 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7890 return False;
7891 ftp_port = urlparts.port;
7892 if(urlparts.port is None):
7893 ftp_port = 21;
7894 try:
7895 ftp.connect(urlparts.hostname, ftp_port);
7896 except socket.gaierror:
7897 log.info("Error With URL "+url);
7898 return False;
7899 except socket.timeout:
7900 log.info("Error With URL "+url);
7901 return False;
7902 ftp.login(urlparts.username, urlparts.password);
7903 if(urlparts.scheme=="ftps"):
7904 ftp.prot_p();
7905 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7906 ftp.close();
7907 ftpfile.seek(0, 0);
7908 return ftpfile;
7909
7910 def upload_file_to_ftp_string(ftpstring, url):
7911 ftpfileo = BytesIO(ftpstring);
7912 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7913 ftpfileo.close();
7914 return ftpfile;
7915
7916 class RawIteratorWrapper:
7917 def __init__(self, iterator):
7918 self.iterator = iterator;
7919 self.buffer = b"";
7920 self._iterator_exhausted = False;
7921 def read(self, size=-1):
7922 if self._iterator_exhausted:
7923 return b'';
7924 while size < 0 or len(self.buffer) < size:
7925 try:
7926 chunk = next(self.iterator);
7927 self.buffer += chunk;
7928 except StopIteration:
7929 self._iterator_exhausted = True;
7930 break;
7931 if size < 0:
7932 size = len(self.buffer);
7933 result, self.buffer = self.buffer[:size], self.buffer[size:];
7934 return result;
7935
7936 def download_file_from_http_file(url, headers=None, usehttp=__use_http_lib__):
7937 if headers is None:
7938 headers = {};
7939 # Parse the URL to extract username and password if present
7940 urlparts = urlparse(url);
7941 username = urlparts.username;
7942 password = urlparts.password;
7943 # Rebuild the URL without the username and password
7944 netloc = urlparts.hostname;
7945 if urlparts.scheme == "sftp":
7946 if __use_pysftp__:
7947 return download_file_from_pysftp_file(url);
7948 else:
7949 return download_file_from_sftp_file(url);
7950 elif urlparts.scheme == "ftp" or urlparts.scheme == "ftps":
7951 return download_file_from_ftp_file(url);
7952 if urlparts.port:
7953 netloc += ':' + str(urlparts.port);
7954 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7955 # Create a temporary file object
7956 httpfile = BytesIO();
7957 if usehttp == 'requests' and haverequests:
7958 # Use the requests library if selected and available
7959 if username and password:
7960 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7961 else:
7962 response = requests.get(rebuilt_url, headers=headers, stream=True);
7963 response.raw.decode_content = True
7964 shutil.copyfileobj(response.raw, httpfile);
7965 elif usehttp == 'httpx' and havehttpx:
7966 # Use httpx if selected and available
7967 with httpx.Client(follow_redirects=True) as client:
7968 if username and password:
7969 response = client.get(rebuilt_url, headers=headers, auth=(username, password));
7970 else:
7971 response = client.get(rebuilt_url, headers=headers);
7972 raw_wrapper = RawIteratorWrapper(response.iter_bytes());
7973 shutil.copyfileobj(raw_wrapper, httpfile);
7974 else:
7975 # Use urllib as a fallback
7976 # Build a Request object for urllib
7977 request = Request(rebuilt_url, headers=headers);
7978 # Create an opener object for handling URLs
7979 if username and password:
7980 # Create a password manager
7981 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7982 # Add the username and password
7983 password_mgr.add_password(None, rebuilt_url, username, password);
7984 # Create an authentication handler using the password manager
7985 auth_handler = HTTPBasicAuthHandler(password_mgr);
7986 # Build the opener with the authentication handler
7987 opener = build_opener(auth_handler);
7988 else:
7989 opener = build_opener();
7990 response = opener.open(request);
7991 shutil.copyfileobj(response, httpfile);
7992 # Reset file pointer to the start
7993 httpfile.seek(0, 0);
7994 # Return the temporary file object
7995 return httpfile;
7996
7997 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt, usehttp=__use_http_lib__):
7998 httpfile = download_file_from_http_file(url, headers, usehttp);
7999 return ftpfile.read();
8000
8001 if(haveparamiko):
8002 def download_file_from_sftp_file(url):
8003 urlparts = urlparse(url);
8004 file_name = os.path.basename(urlparts.path);
8005 file_dir = os.path.dirname(urlparts.path);
8006 sftp_port = urlparts.port;
8007 if(urlparts.port is None):
8008 sftp_port = 22;
8009 else:
8010 sftp_port = urlparts.port;
8011 if(urlparts.username is not None):
8012 sftp_username = urlparts.username;
8013 else:
8014 sftp_username = "anonymous";
8015 if(urlparts.password is not None):
8016 sftp_password = urlparts.password;
8017 elif(urlparts.password is None and urlparts.username=="anonymous"):
8018 sftp_password = "anonymous";
8019 else:
8020 sftp_password = "";
8021 if(urlparts.scheme=="ftp"):
8022 return download_file_from_ftp_file(url);
8023 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8024 return download_file_from_http_file(url);
8025 if(urlparts.scheme!="sftp"):
8026 return False;
8027 ssh = paramiko.SSHClient();
8028 ssh.load_system_host_keys();
8029 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
8030 try:
8031 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8032 except paramiko.ssh_exception.SSHException:
8033 return False;
8034 except socket.gaierror:
8035 log.info("Error With URL "+url);
8036 return False;
8037 except socket.timeout:
8038 log.info("Error With URL "+url);
8039 return False;
8040 sftp = ssh.open_sftp();
8041 sftpfile = BytesIO();
8042 sftp.getfo(urlparts.path, sftpfile);
8043 sftp.close();
8044 ssh.close();
8045 sftpfile.seek(0, 0);
8046 return sftpfile;
8047 else:
8048 def download_file_from_sftp_file(url):
8049 return False;
8050
8051 if(haveparamiko):
8052 def download_file_from_sftp_string(url):
8053 sftpfile = download_file_from_sftp_file(url);
8054 return sftpfile.read();
8055 else:
8056 def download_file_from_ftp_string(url):
8057 return False;
8058
8059 if(haveparamiko):
8060 def upload_file_to_sftp_file(sftpfile, url):
8061 urlparts = urlparse(url);
8062 file_name = os.path.basename(urlparts.path);
8063 file_dir = os.path.dirname(urlparts.path);
8064 sftp_port = urlparts.port;
8065 if(urlparts.port is None):
8066 sftp_port = 22;
8067 else:
8068 sftp_port = urlparts.port;
8069 if(urlparts.username is not None):
8070 sftp_username = urlparts.username;
8071 else:
8072 sftp_username = "anonymous";
8073 if(urlparts.password is not None):
8074 sftp_password = urlparts.password;
8075 elif(urlparts.password is None and urlparts.username=="anonymous"):
8076 sftp_password = "anonymous";
8077 else:
8078 sftp_password = "";
8079 if(urlparts.scheme=="ftp"):
8080 return upload_file_to_ftp_file(url);
8081 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8082 return False;
8083 if(urlparts.scheme!="sftp"):
8084 return False;
8085 ssh = paramiko.SSHClient();
8086 ssh.load_system_host_keys();
8087 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
8088 try:
8089 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8090 except paramiko.ssh_exception.SSHException:
8091 return False;
8092 except socket.gaierror:
8093 log.info("Error With URL "+url);
8094 return False;
8095 except socket.timeout:
8096 log.info("Error With URL "+url);
8097 return False;
8098 sftp = ssh.open_sftp();
8099 sftp.putfo(sftpfile, urlparts.path);
8100 sftp.close();
8101 ssh.close();
8102 sftpfile.seek(0, 0);
8103 return sftpfile;
8104 else:
8105 def upload_file_to_sftp_file(sftpfile, url):
8106 return False;
8107
8108 if(haveparamiko):
8109 def upload_file_to_sftp_string(sftpstring, url):
8110 sftpfileo = BytesIO(sftpstring);
8111 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
8112 sftpfileo.close();
8113 return sftpfile;
8114 else:
8115 def upload_file_to_sftp_string(url):
8116 return False;
8117
8118 if(havepysftp):
8119 def download_file_from_pysftp_file(url):
8120 urlparts = urlparse(url);
8121 file_name = os.path.basename(urlparts.path);
8122 file_dir = os.path.dirname(urlparts.path);
8123 sftp_port = urlparts.port;
8124 if(urlparts.port is None):
8125 sftp_port = 22;
8126 else:
8127 sftp_port = urlparts.port;
8128 if(urlparts.username is not None):
8129 sftp_username = urlparts.username;
8130 else:
8131 sftp_username = "anonymous";
8132 if(urlparts.password is not None):
8133 sftp_password = urlparts.password;
8134 elif(urlparts.password is None and urlparts.username=="anonymous"):
8135 sftp_password = "anonymous";
8136 else:
8137 sftp_password = "";
8138 if(urlparts.scheme=="ftp"):
8139 return download_file_from_ftp_file(url);
8140 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8141 return download_file_from_http_file(url);
8142 if(urlparts.scheme!="sftp"):
8143 return False;
8144 try:
8145 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8146 except paramiko.ssh_exception.SSHException:
8147 return False;
8148 except socket.gaierror:
8149 log.info("Error With URL "+url);
8150 return False;
8151 except socket.timeout:
8152 log.info("Error With URL "+url);
8153 return False;
8154 sftp = ssh.open_sftp();
8155 sftpfile = BytesIO();
8156 sftp.getfo(urlparts.path, sftpfile);
8157 sftp.close();
8158 ssh.close();
8159 sftpfile.seek(0, 0);
8160 return sftpfile;
8161 else:
8162 def download_file_from_pysftp_file(url):
8163 return False;
8164
8165 if(havepysftp):
8166 def download_file_from_pysftp_string(url):
8167 sftpfile = download_file_from_pysftp_file(url);
8168 return sftpfile.read();
8169 else:
8170 def download_file_from_ftp_string(url):
8171 return False;
8172
8173 if(havepysftp):
8174 def upload_file_to_pysftp_file(sftpfile, url):
8175 urlparts = urlparse(url);
8176 file_name = os.path.basename(urlparts.path);
8177 file_dir = os.path.dirname(urlparts.path);
8178 sftp_port = urlparts.port;
8179 if(urlparts.port is None):
8180 sftp_port = 22;
8181 else:
8182 sftp_port = urlparts.port;
8183 if(urlparts.username is not None):
8184 sftp_username = urlparts.username;
8185 else:
8186 sftp_username = "anonymous";
8187 if(urlparts.password is not None):
8188 sftp_password = urlparts.password;
8189 elif(urlparts.password is None and urlparts.username=="anonymous"):
8190 sftp_password = "anonymous";
8191 else:
8192 sftp_password = "";
8193 if(urlparts.scheme=="ftp"):
8194 return upload_file_to_ftp_file(url);
8195 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
8196 return False;
8197 if(urlparts.scheme!="sftp"):
8198 return False;
8199 try:
8200 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
8201 except paramiko.ssh_exception.SSHException:
8202 return False;
8203 except socket.gaierror:
8204 log.info("Error With URL "+url);
8205 return False;
8206 except socket.timeout:
8207 log.info("Error With URL "+url);
8208 return False;
8209 sftp = ssh.open_sftp();
8210 sftp.putfo(sftpfile, urlparts.path);
8211 sftp.close();
8212 ssh.close();
8213 sftpfile.seek(0, 0);
8214 return sftpfile;
8215 else:
8216 def upload_file_to_pysftp_file(sftpfile, url):
8217 return False;
8218
8219 if(havepysftp):
8220 def upload_file_to_pysftp_string(sftpstring, url):
8221 sftpfileo = BytesIO(sftpstring);
8222 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
8223 sftpfileo.close();
8224 return sftpfile;
8225 else:
8226 def upload_file_to_pysftp_string(url):
8227 return False;
8228
8229 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt, usehttp=__use_http_lib__):
8230 urlparts = urlparse(url);
8231 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8232 return download_file_from_http_file(url, headers, usehttp);
8233 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8234 return download_file_from_ftp_file(url);
8235 elif(urlparts.scheme=="sftp"):
8236 if(__use_pysftp__ and havepysftp):
8237 return download_file_from_pysftp_file(url);
8238 else:
8239 return download_file_from_sftp_file(url);
8240 else:
8241 return False;
8242 return False;
8243
8244 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
8245 formatspecs = FormatSpecsListToDict(formatspecs);
8246 fp = download_file_from_internet_file(url);
8247 fp = UncompressArchiveFile(fp, formatspecs);
8248 fp.seek(0, 0);
8249 if(not fp):
8250 return False;
8251 return fp;
8252
8253 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
8254 urlparts = urlparse(url);
8255 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8256 return download_file_from_http_string(url, headers);
8257 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8258 return download_file_from_ftp_string(url);
8259 elif(urlparts.scheme=="sftp"):
8260 if(__use_pysftp__ and havepysftp):
8261 return download_file_from_pysftp_string(url);
8262 else:
8263 return download_file_from_sftp_string(url);
8264 else:
8265 return False;
8266 return False;
8267
8268 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
8269 formatspecs = FormatSpecsListToDict(formatspecs);
8270 fp = download_file_from_internet_string(url);
8271 fp = UncompressArchiveFile(fp, formatspecs);
8272 fp.seek(0, 0);
8273 if(not fp):
8274 return False;
8275 return fp;
8276
8277 def upload_file_to_internet_file(ifp, url):
8278 urlparts = urlparse(url);
8279 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8280 return False;
8281 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8282 return upload_file_to_ftp_file(ifp, url);
8283 elif(urlparts.scheme=="sftp"):
8284 if(__use_pysftp__ and havepysftp):
8285 return upload_file_to_pysftp_file(ifp, url);
8286 else:
8287 return upload_file_to_sftp_file(ifp, url);
8288 else:
8289 return False;
8290 return False;
8291
8292 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
8293 formatspecs = FormatSpecsListToDict(formatspecs);
8294 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
8295 if(not catfileout):
8296 return False;
8297 catfp.seek(0, 0);
8298 upload_file_to_internet_file(catfp, outfile);
8299 return True;
8300
8301 def upload_file_to_internet_string(ifp, url):
8302 urlparts = urlparse(url);
8303 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8304 return False;
8305 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8306 return upload_file_to_ftp_string(ifp, url);
8307 elif(urlparts.scheme=="sftp"):
8308 if(__use_pysftp__ and havepysftp):
8309 return upload_file_to_pysftp_string(ifp, url);
8310 else:
8311 return upload_file_to_sftp_string(ifp, url);
8312 else:
8313 return False;
8314 return False;
8315
8316 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
8317 formatspecs = FormatSpecsListToDict(formatspecs);
8318 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
8319 if(not catfileout):
8320 return False;
8321 catfp.seek(0, 0);
8322 upload_file_to_internet_file(catfp, outfile);
8323 return True;
8324
8325 try:
8326 if(hasattr(shutil, "register_archive_format")):
8327 # Register the packing format
8328 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
8329 except shutil.RegistryError:
8330 pass;
8331
8332 try:
8333 if(hasattr(shutil, "register_unpack_format")):
8334 # Register the unpacking format
8335 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
8336 except shutil.RegistryError:
8337 pass;
A tar like file format name catfile after unix cat command (concatenate files).