2 # -*- coding: utf-8 -*-
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/31/2024 Ver. 0.12.0 RC 1 - Author: cooldude2k $
20 from __future__
import absolute_import
, division
, print_function
, unicode_literals
;
21 import io
, os
, re
, sys
, time
, stat
, zlib
, base64
, shutil
, socket
, hashlib
, datetime
, logging
, binascii
, zipfile
, platform
;
23 from backports
import tempfile
;
29 from ftplib
import FTP
, FTP_TLS
;
32 from ftplib
import FTP
;
35 import simplejson
as json
;
41 from urllib
.parse
import urlparse
, urlunparse
;
43 from urlparse
import urlparse
, urlunparse
;
45 # Windows-specific setup
47 if sys
.version_info
[0] == 2:
49 sys
.stdout
= codecs
.getwriter('utf-8')(sys
.stdout
);
50 sys
.stderr
= codecs
.getwriter('utf-8')(sys
.stderr
);
52 sys
.stdout
= io
.TextIOWrapper(sys
.stdout
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
53 sys
.stderr
= io
.TextIOWrapper(sys
.stderr
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
55 hashlib_guaranteed
= False;
57 os
.environ
["PYTHONIOENCODING"] = "UTF-8";
58 os
.environ
["LC_CTYPE"] = "UTF-8";
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys
.version_info
[0] == 2:
64 sys
.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
70 from zlib
import crc32
;
72 from binascii
import crc32
;
74 # Define FileNotFoundError for Python 2
78 FileNotFoundError
= IOError;
81 rarfile_support
= False;
84 rarfile_support
= True;
89 py7zr_support
= False;
98 from xtarfile
import is_tarfile
;
101 from safetar
import is_tarfile
;
103 from tarfile
import is_tarfile
;
107 import xtarfile
as tarfile
;
110 import safetar
as tarfile
;
115 haveparamiko
= False;
131 haverequests
= False;
136 logging
.getLogger("urllib3").setLevel(logging
.WARNING
);
140 # HTTP and URL parsing
142 from urllib
.request
import Request
, build_opener
, HTTPBasicAuthHandler
;
143 from urllib
.parse
import urlparse
;
145 from urllib2
import Request
, build_opener
, HTTPBasicAuthHandler
;
146 from urlparse
import urlparse
;
148 # StringIO and BytesIO
150 from io
import StringIO
, BytesIO
;
153 from cStringIO
import StringIO
;
154 from cStringIO
import StringIO
as BytesIO
;
156 from StringIO
import StringIO
;
157 from StringIO
import StringIO
as BytesIO
;
159 __use_pysftp__
= False;
161 __use_pysftp__
= False;
162 __file_format_name__
= "CatFile";
163 __program_name__
= "Py"+__file_format_name__
;
164 __file_format_lower__
= __file_format_name__
.lower();
165 __file_format_magic__
= __file_format_name__
;
166 __file_format_len__
= len(__file_format_magic__
);
167 __file_format_hex__
= binascii
.hexlify(__file_format_magic__
.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__
= "\x00";
169 __file_format_ver__
= "001";
170 __use_new_style__
= True;
171 __use_advanced_list__
= True;
172 __use_alt_inode__
= False;
173 __file_format_extension__
= ".cat";
174 __file_format_list__
= [__file_format_name__
, __file_format_magic__
, __file_format_lower__
, __file_format_len__
, __file_format_hex__
, __file_format_delimiter__
, __file_format_ver__
, __use_new_style__
, __use_advanced_list__
, __use_alt_inode__
];
175 __file_format_dict__
= {'format_name': __file_format_name__
, 'format_magic': __file_format_magic__
, 'format_lower': __file_format_lower__
, 'format_len': __file_format_len__
, 'format_hex': __file_format_hex__
, 'format_delimiter': __file_format_delimiter__
, 'format_ver': __file_format_ver__
, 'new_style': __use_new_style__
, 'use_advanced_list': __use_advanced_list__
, 'use_alt_inode': __use_alt_inode__
};
176 __project__
= __program_name__
;
177 __project_url__
= "https://github.com/GameMaker2k/PyCatFile";
178 __version_info__
= (0, 12, 0, "RC 1", 1);
179 __version_date_info__
= (2024, 5, 31, "RC 1", 1);
180 __version_date__
= str(__version_date_info__
[0]) + "." + str(__version_date_info__
[1]).zfill(2) + "." + str(__version_date_info__
[2]).zfill(2);
181 __revision__
= __version_info__
[3];
182 __revision_id__
= "$Id$";
183 if(__version_info__
[4] is not None):
184 __version_date_plusrc__
= __version_date__
+ "-" + str(__version_date_info__
[4]);
185 if(__version_info__
[4] is None):
186 __version_date_plusrc__
= __version_date__
;
187 if(__version_info__
[3] is not None):
188 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]) + " " + str(__version_info__
[3]);
189 if(__version_info__
[3] is None):
190 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]);
192 PyBitness
= platform
.architecture();
193 if(PyBitness
=="32bit" or PyBitness
=="32"):
195 elif(PyBitness
=="64bit" or PyBitness
=="64"):
200 geturls_ua_pycatfile_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
201 if(platform
.python_implementation()!=""):
202 py_implementation
= platform
.python_implementation();
203 if(platform
.python_implementation()==""):
204 py_implementation
= "Python";
205 geturls_ua_pycatfile_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
206 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
207 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
208 geturls_headers_pycatfile_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
209 geturls_headers_pycatfile_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
210 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 def CompressionSupport():
214 compression_list
= [];
217 compression_list
.append("gz");
218 compression_list
.append("gzip");
223 compression_list
.append("bz2");
224 compression_list
.append("bzip2");
229 compression_list
.append("lz4");
234 compression_list
.append("lzo");
235 compression_list
.append("lzop");
240 compression_list
.append("zstd");
241 compression_list
.append("zstandard");
246 compression_list
.append("lzma");
247 compression_list
.append("xz");
251 from backports
import lzma
;
252 compression_list
.append("lzma");
253 compression_list
.append("xz");
256 return compression_list
;
258 compressionsupport
= CompressionSupport();
259 compressionlist
= ['auto'];
260 compressionlistalt
= [];
263 if('gzip' in compressionsupport
):
264 compressionlist
.append('gzip');
265 compressionlistalt
.append('gzip');
266 outextlist
.append('gz');
267 outextlistwd
.append('.gz');
268 if('bzip2' in compressionsupport
):
269 compressionlist
.append('bzip2');
270 compressionlistalt
.append('bzip2');
271 outextlist
.append('bz2');
272 outextlistwd
.append('.bz2');
273 if('zstd' in compressionsupport
):
274 compressionlist
.append('zstd');
275 compressionlistalt
.append('zstd');
276 outextlist
.append('zst');
277 outextlistwd
.append('.zst');
278 if('lz4' in compressionsupport
):
279 compressionlist
.append('lz4');
280 compressionlistalt
.append('lz4');
281 outextlist
.append('lz4');
282 outextlistwd
.append('.lz4');
283 if('lzo' in compressionsupport
):
284 compressionlist
.append('lzo');
285 compressionlistalt
.append('lzo');
286 outextlist
.append('lzo');
287 outextlistwd
.append('.lzo');
288 if('lzop' in compressionsupport
):
289 compressionlist
.append('lzop');
290 compressionlistalt
.append('lzop');
291 outextlist
.append('lzop');
292 outextlistwd
.append('.lzop');
293 if('lzma' in compressionsupport
):
294 compressionlist
.append('lzma');
295 compressionlistalt
.append('lzma');
296 outextlist
.append('lzma');
297 outextlistwd
.append('.lzma');
298 if('xz' in compressionsupport
):
299 compressionlist
.append('xz');
300 compressionlistalt
.append('xz');
301 outextlist
.append('xz');
302 outextlistwd
.append('.xz');
304 tarfile_mimetype
= "application/tar";
305 tarfile_tar_mimetype
= tarfile_mimetype
;
306 zipfile_mimetype
= "application/zip";
307 zipfile_zip_mimetype
= zipfile_mimetype
;
308 rarfile_mimetype
= "application/rar";
309 rarfile_rar_mimetype
= rarfile_mimetype
;
310 archivefile_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"";
311 archivefile_cat_mimetype
= archivefile_mimetype
;
312 archivefile_gzip_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+gzip";
313 archivefile_gz_mimetype
= archivefile_gzip_mimetype
;
314 archivefile_bzip2_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+bzip2";
315 archivefile_bz2_mimetype
= archivefile_bzip2_mimetype
;
316 archivefile_lz4_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lz4";
317 archivefile_lzop_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzop";
318 archivefile_lzo_mimetype
= archivefile_lzop_mimetype
;
319 archivefile_zstandard_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+zstandard";
320 archivefile_zstd_mimetype
= archivefile_zstandard_mimetype
;
321 archivefile_lzma_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzma";
322 archivefile_xz_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+xz";
323 archivefile_extensions
= [__file_format_extension__
, __file_format_extension__
+".gz", __file_format_extension__
+".bz2", __file_format_extension__
+".zst", __file_format_extension__
+".lz4", __file_format_extension__
+".lzo", __file_format_extension__
+".lzop", __file_format_extension__
+".lzma", __file_format_extension__
+".xz"];
325 if __name__
== "__main__":
327 curscrpath
= os
.path
.dirname(sys
.argv
[0]);
331 curscrpath
= curscrpath
.replace(os
.sep
, "/");
332 curscrpath
= curscrpath
+ "/";
333 scrfile
= curscrpath
+ "catfile.py";
334 if(os
.path
.exists(scrfile
) and os
.path
.isfile(scrfile
)):
335 scrcmd
= subprocess
.Popen([sys
.executable
, scrfile
] + sys
.argv
[1:]);
338 def VerbosePrintOut(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
344 "warning": logging
.warning
,
345 "error": logging
.error
,
346 "critical": logging
.critical
,
347 "exception": logging
.exception
,
348 "logalt": lambda x
: logging
.log(dgblevel
, x
),
349 "debug": logging
.debug
351 log_function
= log_functions
.get(outtype
);
353 log_function(dbgtxt
);
357 def VerbosePrintOutReturn(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
358 VerbosePrintOut(dbgtxt
, outtype
, dbgenable
, dgblevel
);
361 def RemoveWindowsPath(dpath
):
365 dpath
= dpath
.replace(os
.path
.sep
, "/");
366 dpath
= dpath
.rstrip("/");
367 if(dpath
=="." or dpath
==".."):
371 def NormalizeRelativePath(inpath
):
372 inpath
= RemoveWindowsPath(inpath
);
373 if(os
.path
.isabs(inpath
)):
376 if(inpath
.startswith("./") or inpath
.startswith("../")):
379 outpath
= "./" + inpath
;
382 def PrependPath(base_dir
, child_path
):
383 # Check if base_dir is None or empty, if so, return child_path as is
386 # Ensure base_dir ends with exactly one slash
387 if not base_dir
.endswith('/'):
389 # Check if child_path starts with ./ or ../ (indicating a relative path)
390 if child_path
.startswith('./') or child_path
.startswith('../'):
391 # For relative paths, we don't alter the child_path
392 return base_dir
+ child_path
;
394 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
395 return base_dir
+ child_path
.lstrip('/');
397 def ListDir(dirpath
, followlink
=False, duplicates
=False):
398 if isinstance(dirpath
, (list, tuple, )):
399 dirpath
= list(filter(None, dirpath
));
400 elif isinstance(dirpath
, str):
401 dirpath
= list(filter(None, [dirpath
]));
403 fs_encoding
= sys
.getfilesystemencoding();
404 for mydirfile
in dirpath
:
405 if not os
.path
.exists(mydirfile
):
407 mydirfile
= NormalizeRelativePath(mydirfile
);
408 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
):
409 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
410 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
411 for root
, dirs
, filenames
in os
.walk(mydirfile
):
413 dpath
= RemoveWindowsPath(dpath
);
414 if fs_encoding
!= 'utf-8':
415 dpath
= dpath
.encode(fs_encoding
).decode('utf-8');
416 if dpath
not in retlist
and not duplicates
:
417 retlist
.append(dpath
);
419 retlist
.append(dpath
);
420 for file in filenames
:
421 fpath
= os
.path
.join(root
, file);
422 fpath
= RemoveWindowsPath(fpath
);
423 if fs_encoding
!= 'utf-8':
424 fpath
= fpath
.encode(fs_encoding
).decode('utf-8');
425 if fpath
not in retlist
and not duplicates
:
426 retlist
.append(fpath
);
428 retlist
.append(fpath
);
430 path
= RemoveWindowsPath(mydirfile
);
431 if fs_encoding
!= 'utf-8':
432 path
= path
.encode(fs_encoding
).decode('utf-8');
433 retlist
.append(path
);
436 def ListDirAdvanced(dirpath
, followlink
=False, duplicates
=False):
437 if isinstance(dirpath
, (list, tuple, )):
438 dirpath
= list(filter(None, dirpath
));
439 elif isinstance(dirpath
, str):
440 dirpath
= list(filter(None, [dirpath
]));
442 fs_encoding
= sys
.getfilesystemencoding();
443 for mydirfile
in dirpath
:
444 if not os
.path
.exists(mydirfile
):
446 mydirfile
= NormalizeRelativePath(mydirfile
);
447 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
) and followlink
:
448 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
449 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
450 for root
, dirs
, filenames
in os
.walk(mydirfile
):
451 # Sort dirs and filenames alphabetically in place
452 dirs
.sort(key
=lambda x
: x
.lower());
453 filenames
.sort(key
=lambda x
: x
.lower());
454 dpath
= RemoveWindowsPath(root
);
455 if fs_encoding
!= 'utf-8':
456 dpath
= dpath
.encode(fs_encoding
).decode('utf-8');
457 if not duplicates
and dpath
not in retlist
:
458 retlist
.append(dpath
);
460 retlist
.append(dpath
);
461 for file in filenames
:
462 fpath
= os
.path
.join(root
, file);
463 fpath
= RemoveWindowsPath(fpath
);
464 if fs_encoding
!= 'utf-8':
465 fpath
= fpath
.encode(fs_encoding
).decode('utf-8');
466 if not duplicates
and fpath
not in retlist
:
467 retlist
.append(fpath
);
469 retlist
.append(fpath
);
471 path
= RemoveWindowsPath(mydirfile
);
472 if fs_encoding
!= 'utf-8':
473 path
= path
.encode(fs_encoding
).decode('utf-8');
474 retlist
.append(path
);
477 def create_alias_function(prefix
, base_name
, suffix
, target_function
):
478 # Define a new function that wraps the target function
479 def alias_function(*args
, **kwargs
):
480 return target_function(*args
, **kwargs
);
481 # Create the function name by combining the prefix, base name, and the suffix
482 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
);
483 # Add the new function to the global namespace
484 globals()[function_name
] = alias_function
;
486 def create_alias_function_alt(prefix
, base_name
, suffix
, target_function
):
487 # Create the function name by combining the prefix, base name, and the suffix
488 # Use the format method for string formatting, compatible with Python 2 and 3
489 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
)
490 # Add the new function (alias of the target_function) to the global namespace
491 # This line is compatible as-is with both Python 2 and 3
492 globals()[function_name
] = target_function
494 def FormatSpecsListToDict(formatspecs
=__file_format_list__
):
495 if(isinstance(formatspecs
, (list, tuple, ))):
496 return {'format_name': formatspecs
[0], 'format_magic': formatspecs
[1], 'format_lower': formatspecs
[2], 'format_len': formatspecs
[3], 'format_hex': formatspecs
[4], 'format_delimiter': formatspecs
[5], 'format_ver': formatspecs
[6], 'new_style': formatspecs
[7], 'use_advanced_list': formatspecs
[8], 'use_alt_inode': formatspecs
[9]};
497 elif(isinstance(formatspecs
, (dict, ))):
500 return __file_format_dict__
;
501 return __file_format_dict__
;
503 def TarFileCheck(infile
):
505 if is_tarfile(infile
):
511 # Check if the input is a file object
512 if hasattr(infile
, 'read'):
513 # Save the current file position
514 current_position
= infile
.tell();
515 # Attempt to open the file object as a tar file
516 tar
= tarfile
.open(fileobj
=infile
);
518 # Restore the file position
519 infile
.seek(current_position
);
522 # Assume it's a filename
523 tar
= tarfile
.open(name
=infile
);
526 except tarfile
.TarError
:
529 # initial_value can be 0xFFFF or 0x0000
530 def crc16_ansi(msg
, initial_value
=0xFFFF):
531 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
532 poly
= 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
533 crc
= initial_value
; # Initial value
535 crc ^
= b
<< 8; # XOR byte into CRC top byte
536 for _
in range(8): # Process each bit
537 if crc
& 0x8000: # If the top bit is set
538 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
540 crc
= crc
<< 1; # Just shift left
541 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
544 # initial_value can be 0xFFFF or 0x0000
545 def crc16_ibm(msg
, initial_value
=0xFFFF):
546 return crc16_ansi(msg
, initial_value
);
548 # initial_value is 0xFFFF
550 return crc16_ansi(msg
, 0xFFFF);
552 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
553 def crc16_ccitt(msg
, initial_value
=0xFFFF):
554 # CRC-16-CCITT polynomial
555 poly
= 0x1021; # Polynomial for CRC-16-CCITT
556 # Use the specified initial value
559 crc ^
= b
<< 8; # XOR byte into CRC top byte
560 for _
in range(8): # Process each bit
561 if crc
& 0x8000: # If the top bit is set
562 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
564 crc
= crc
<< 1; # Just shift left
565 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
568 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
569 def crc64_ecma(msg
, initial_value
=0x0000000000000000):
570 # CRC-64-ECMA polynomial and initial value
571 poly
= 0x42F0E1EBA9EA3693;
572 crc
= initial_value
; # Initial value for CRC-64-ECMA
574 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
575 for _
in range(8): # Process each bit
576 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
577 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
579 crc
<<= 1; # Just shift left if the MSB is 0
580 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
583 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
584 def crc64_iso(msg
, initial_value
=0xFFFFFFFFFFFFFFFF):
585 # CRC-64-ISO polynomial and initial value
586 poly
= 0x000000000000001B;
587 crc
= initial_value
; # Common initial value for CRC-64-ISO
589 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
590 for _
in range(8): # Process each bit
591 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
592 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
594 crc
<<= 1; # Just shift left if the MSB is 0
595 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
598 def GetDataFromArray(data
, path
, default
=None):
602 element
= element
[key
];
604 except (KeyError, TypeError, IndexError):
607 def GetDataFromArrayAlt(structure
, path
, default
=None):
610 if isinstance(element
, dict) and key
in element
:
611 element
= element
[key
];
612 elif isinstance(element
, list) and isinstance(key
, int) and -len(element
) <= key
< len(element
):
613 element
= element
[key
];
618 def GetHeaderChecksum(inlist
=[], checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
619 formatspecs
= FormatSpecsListToDict(formatspecs
);
620 fileheader
= AppendNullBytes(inlist
, formatspecs
['format_delimiter']) if isinstance(inlist
, list) else AppendNullByte(inlist
, formatspecs
['format_delimiter']);
622 fileheader
= fileheader
.encode('UTF-8');
624 "crc16": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
625 "crc16_ansi": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
626 "crc16_ibm": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
627 "crc16_ccitt": lambda data
: format(crc16_ccitt(data
) & 0xffff, '04x').lower(),
628 "adler32": lambda data
: format(zlib
.adler32(data
) & 0xffffffff, '08x').lower(),
629 "crc32": lambda data
: format(crc32(data
) & 0xffffffff, '08x').lower(),
630 "crc64_ecma": lambda data
: format(crc64_ecma(data
) & 0xffffffffffffffff, '016x').lower(),
631 "crc64": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
632 "crc64_iso": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
634 if checksumtype
in checksum_methods
:
635 return checksum_methods
[checksumtype
](fileheader
);
636 elif CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
):
637 checksumoutstr
= hashlib
.new(checksumtype
);
638 checksumoutstr
.update(fileheader
);
639 return checksumoutstr
.hexdigest().lower();
640 return format(0, 'x').lower();
642 def GetFileChecksum(instr
, checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
643 formatspecs
= FormatSpecsListToDict(formatspecs
);
645 instr
= instr
.encode('UTF-8');
647 "crc16": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
648 "crc16_ansi": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
649 "crc16_ibm": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
650 "crc16_ccitt": lambda data
: format(crc16_ccitt(data
) & 0xffff, '04x').lower(),
651 "adler32": lambda data
: format(zlib
.adler32(data
) & 0xffffffff, '08x').lower(),
652 "crc32": lambda data
: format(crc32(data
) & 0xffffffff, '08x').lower(),
653 "crc64_ecma": lambda data
: format(crc64_ecma(data
) & 0xffffffffffffffff, '016x').lower(),
654 "crc64": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
655 "crc64_iso": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
657 if checksumtype
in checksum_methods
:
658 return checksum_methods
[checksumtype
](instr
);
659 elif CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
):
660 checksumoutstr
= hashlib
.new(checksumtype
);
661 checksumoutstr
.update(instr
);
662 return checksumoutstr
.hexdigest().lower();
663 return format(0, 'x').lower();
665 def ValidateHeaderChecksum(inlist
=[], checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
666 formatspecs
= FormatSpecsListToDict(formatspecs
);
667 catfileheadercshex
= GetHeaderChecksum(inlist
, checksumtype
, True, formatspecs
).lower();
668 return inchecksum
.lower() == catfileheadercshex
;
670 def ValidateFileChecksum(infile
, checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
671 formatspecs
= FormatSpecsListToDict(formatspecs
);
672 catinfilecshex
= GetFileChecksum(infile
, checksumtype
, True, formatspecs
).lower();
673 return inchecksum
.lower() == catinfilecshex
;
675 def ReadTillNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
678 nullbyte
= delimiter
.encode("UTF-8");
680 curbyte
= fp
.read(1);
681 if(curbyte
==nullbyte
or not curbyte
):
683 curfullbyte
= curfullbyte
+ curbyte
;
684 return curfullbyte
.decode('UTF-8');
686 def ReadUntilNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
687 return ReadTillNullByteOld(fp
, delimiter
);
689 def ReadTillNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
690 delimiter
= delimiter
.encode('UTF-8') # Ensure the delimiter is in bytes
691 buffer = bytearray();
693 delimiter_length
= len(delimiter
);
695 chunk
= fp
.read(chunk_size
)
697 # End of file reached without finding the delimiter
699 buffer.extend(chunk
);
700 total_read
+= len(chunk
);
701 if delimiter
in buffer:
702 # Delimiter found, calculate where to reset the file pointer
703 index
= buffer.find(delimiter
);
704 # Calculate how many extra bytes were read after the delimiter
705 extra_bytes_read
= len(buffer) - (index
+ delimiter_length
);
706 # Move the file pointer back to just after the delimiter
707 fp
.seek(-extra_bytes_read
, 1);
708 buffer = buffer[:index
];
710 if total_read
>= max_read
:
711 # Stop reading if max limit is reached to prevent excessive memory usage
712 raise MemoryError("Maximum read limit reached without finding the delimiter.");
713 # Check for incomplete UTF-8 sequences at the end of the buffer
714 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
715 # This suggests that the last byte might be the start of a multi-byte character
716 # Try to read one more byte to complete the character
717 extra_byte
= fp
.read(1);
719 buffer.extend(extra_byte
);
721 # No more data available
724 return buffer.decode('UTF-8', errors
='replace');
725 except UnicodeDecodeError:
726 return buffer.decode('UTF-8', errors
='replace');
728 def ReadUntilNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
729 return ReadTillNullByteAlt(fp
, delimiter
, chunk_size
, max_read
);
731 def ReadTillNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
732 curfullbyte
= bytearray();
733 nullbyte
= delimiter
.encode("UTF-8");
734 total_read
= 0; # Track the total number of bytes read
736 curbyte
= fp
.read(1);
737 if curbyte
== nullbyte
or not curbyte
:
739 curfullbyte
.extend(curbyte
);
741 if total_read
>= max_read
:
742 raise MemoryError("Maximum read limit reached without finding the delimiter.");
743 # Decode the full byte array to string once out of the loop
745 return curfullbyte
.decode('UTF-8');
746 except UnicodeDecodeError:
747 # Handle potential partial UTF-8 characters
748 for i
in range(1, 4):
750 return curfullbyte
[:-i
].decode('UTF-8');
751 except UnicodeDecodeError:
753 raise; # Re-raise if decoding fails even after trimming
755 def ReadUntilNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
756 return ReadTillNullByte(fp
, delimiter
, max_read
);
758 def ReadTillNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
759 delimiter
= delimiter
.encode('UTF-8'); # Ensure the delimiter is in bytes
760 buffer = bytearray();
762 delimiter_length
= len(delimiter
);
764 while len(results
) < num_delimiters
:
765 chunk
= fp
.read(chunk_size
);
767 # End of file reached; decode whatever is collected if it's the last needed part
769 results
.append(buffer.decode('UTF-8', errors
='replace'));
772 total_read
+= len(chunk
);
773 # Check if we have found the delimiter
774 while delimiter
in buffer:
775 index
= buffer.find(delimiter
);
776 # Decode the section before the delimiter
777 results
.append(buffer[:index
].decode('UTF-8', errors
='replace'));
778 # Remove the processed part from the buffer
779 buffer = buffer[index
+ delimiter_length
:];
780 if len(results
) == num_delimiters
:
781 # If reached the required number of delimiters, adjust the file pointer and stop
782 fp
.seek(-len(buffer), 1);
784 if total_read
>= max_read
:
785 # Stop reading if max limit is reached to prevent excessive memory usage
786 raise MemoryError("Maximum read limit reached without finding the delimiter.");
787 # Check for incomplete UTF-8 sequences at the end of the buffer
788 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
789 # This suggests that the last byte might be the start of a multi-byte character
790 # Try to read one more byte to complete the character
791 extra_byte
= fp
.read(1);
793 buffer.extend(extra_byte
);
795 # No more data available
797 # Process remaining buffer if less than the required number of delimiters were found
798 if len(buffer) > 0 and len(results
) < num_delimiters
:
799 results
.append(buffer.decode('UTF-8', errors
='replace'));
802 def ReadUntilNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
803 return ReadTillNullByteByNum(fp
, delimiter
, num_delimiters
, chunk_size
, max_read
);
805 def SeekToEndOfFile(fp
):
809 if(lasttell
==fp
.tell()):
811 lasttell
= fp
.tell();
814 def ReadFileHeaderData(fp
, rounds
=0, delimiter
=__file_format_dict__
['format_delimiter']):
818 while(rocount
<roend
):
819 HeaderOut
.append(ReadTillNullByte(fp
, delimiter
));
820 rocount
= rocount
+ 1;
823 def ReadFileHeaderDataBySize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
824 headerpresize
= ReadTillNullByte(fp
, delimiter
);
825 headersize
= int(headerpresize
, 16);
828 headercontent
= str(fp
.read(headersize
).decode('UTF-8')).split(delimiter
);
831 roend
= int(len(headercontent
));
832 HeaderOut
= [headerpresize
];
833 while(rocount
<roend
):
834 HeaderOut
.append(headercontent
[rocount
]);
835 rocount
= rocount
+ 1;
838 def ReadFileHeaderDataWoSize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
839 preheaderdata
= ReadFileHeaderData(fp
, 2, delimiter
);
840 headersize
= int(preheaderdata
[0], 16);
841 headernumfields
= int(preheaderdata
[1], 16);
842 if(headersize
<=0 or headernumfields
<=0):
844 headerdata
= ReadTillNullByteByNum(fp
, delimiter
, headernumfields
);
845 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
846 HeaderOut
= preheaderdata
+ headerdata
;
849 def ReadFileHeaderDataBySizeWithContent(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
850 formatspecs
= FormatSpecsListToDict(formatspecs
);
851 delimiter
= formatspecs
['format_delimiter'];
852 fheaderstart
= fp
.tell();
853 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
854 if(len(HeaderOut
)==0):
856 if(re
.findall(r
"^[.|/]", HeaderOut
[3])):
857 fname
= HeaderOut
[3];
859 fname
= "./"+HeaderOut
[3];
860 fcs
= HeaderOut
[-2].lower();
861 fccs
= HeaderOut
[-1].lower();
862 fsize
= int(HeaderOut
[5], 16);
863 fcompression
= HeaderOut
[12];
864 fcsize
= int(HeaderOut
[13], 16);
865 fseeknextfile
= HeaderOut
[25];
866 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-4].lower(), True, formatspecs
);
867 if(fcs
!=newfcs
and not skipchecksum
):
868 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
869 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
871 fhend
= fp
.tell() - 1;
872 fcontentstart
= fp
.tell();
873 fcontents
= BytesIO();
874 if(fsize
>0 and not listonly
):
875 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
876 fcontents
.write(fp
.read(fsize
));
878 fcontents
.write(fp
.read(fcsize
));
879 elif(fsize
>0 and listonly
):
880 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
884 fcontents
.seek(0, 0);
885 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
886 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
887 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
888 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
890 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
893 fcontents
.seek(0, 0);
895 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
896 fcontentend
= fp
.tell();
897 if(re
.findall(r
"^\+([0-9]+)", fseeknextfile
)):
898 fseeknextasnum
= int(fseeknextfile
.replace("+", ""));
899 if(abs(fseeknextasnum
)==0):
901 fp
.seek(fseeknextasnum
, 1);
902 elif(re
.findall(r
"^\-([0-9]+)", fseeknextfile
)):
903 fseeknextasnum
= int(fseeknextfile
);
904 if(abs(fseeknextasnum
)==0):
906 fp
.seek(fseeknextasnum
, 1);
907 elif(re
.findall(r
"^([0-9]+)", fseeknextfile
)):
908 fseeknextasnum
= int(fseeknextfile
);
909 if(abs(fseeknextasnum
)==0):
911 fp
.seek(fseeknextasnum
, 0);
914 HeaderOut
.append(fcontents
);
917 def ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
918 formatspecs
= FormatSpecsListToDict(formatspecs
);
919 delimiter
= formatspecs
['format_delimiter'];
920 fheaderstart
= fp
.tell();
921 if(formatspecs
['new_style']):
922 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
924 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
925 if(len(HeaderOut
)==0):
927 fheadsize
= int(HeaderOut
[0], 16);
928 fnumfields
= int(HeaderOut
[1], 16);
929 ftype
= int(HeaderOut
[2], 16);
930 if(re
.findall(r
"^[.|/]", HeaderOut
[3])):
931 fname
= HeaderOut
[3];
933 fname
= "./"+HeaderOut
[3];
934 fbasedir
= os
.path
.dirname(fname
);
935 flinkname
= HeaderOut
[4];
936 fsize
= int(HeaderOut
[5], 16);
937 fatime
= int(HeaderOut
[6], 16);
938 fmtime
= int(HeaderOut
[7], 16);
939 fctime
= int(HeaderOut
[8], 16);
940 fbtime
= int(HeaderOut
[9], 16);
941 fmode
= int(HeaderOut
[10], 16);
942 fchmode
= stat
.S_IMODE(fmode
);
943 ftypemod
= stat
.S_IFMT(fmode
);
944 fwinattributes
= int(HeaderOut
[11], 16);
945 fcompression
= HeaderOut
[12];
946 fcsize
= int(HeaderOut
[13], 16);
947 fuid
= int(HeaderOut
[14], 16);
948 funame
= HeaderOut
[15];
949 fgid
= int(HeaderOut
[16], 16);
950 fgname
= HeaderOut
[17];
951 fid
= int(HeaderOut
[18], 16);
952 finode
= int(HeaderOut
[19], 16);
953 flinkcount
= int(HeaderOut
[20], 16);
954 fdev_minor
= int(HeaderOut
[21], 16);
955 fdev_major
= int(HeaderOut
[22], 16);
956 frdev_minor
= int(HeaderOut
[23], 16);
957 frdev_major
= int(HeaderOut
[24], 16);
958 fseeknextfile
= HeaderOut
[25];
959 fextrasize
= int(HeaderOut
[26], 16);
960 fextrafields
= int(HeaderOut
[27], 16);
961 extrafieldslist
= [];
963 extraend
= extrastart
+ fextrafields
;
964 extrafieldslist
= [];
965 if(extrastart
<extraend
):
966 extrafieldslist
.append(HeaderOut
[extrastart
]);
967 extrastart
= extrastart
+ 1;
968 fcs
= HeaderOut
[extrastart
+ 2].lower();
969 fccs
= HeaderOut
[extrastart
+ 3].lower();
970 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-4].lower(), True, formatspecs
);
971 if(fcs
!=newfcs
and not skipchecksum
):
972 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
973 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
975 fhend
= fp
.tell() - 1;
976 fcontentstart
= fp
.tell();
977 fcontents
= BytesIO();
978 pyhascontents
= False;
979 if(fsize
>0 and not listonly
):
980 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
981 fcontents
.write(fp
.read(fsize
));
983 fcontents
.write(fp
.read(fcsize
));
984 pyhascontents
= True;
985 elif(fsize
>0 and listonly
):
986 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
990 pyhascontents
= False;
991 fcontents
.seek(0, 0);
992 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
993 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
994 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
995 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
997 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1000 fcontents
.seek(0, 0);
1002 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
1003 fcontents
.seek(0, 0);
1004 fccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
1005 fcontentend
= fp
.tell() - 1;
1006 if(re
.findall(r
"^\+([0-9]+)", fseeknextfile
)):
1007 fseeknextasnum
= int(fseeknextfile
.replace("+", ""));
1008 if(abs(fseeknextasnum
)==0):
1010 fp
.seek(fseeknextasnum
, 1);
1011 elif(re
.findall(r
"^\-([0-9]+)", fseeknextfile
)):
1012 fseeknextasnum
= int(fseeknextfile
);
1013 if(abs(fseeknextasnum
)==0):
1015 fp
.seek(fseeknextasnum
, 1);
1016 elif(re
.findall(r
"^([0-9]+)", fseeknextfile
)):
1017 fseeknextasnum
= int(fseeknextfile
);
1018 if(abs(fseeknextasnum
)==0):
1020 fp
.seek(fseeknextasnum
, 0);
1023 fcontents
.seek(0, 0);
1024 catlist
= {'fheadersize': fheadsize
, 'fhstart': fheaderstart
, 'fhend': fhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': fseeknextfile
, 'fheaderchecksumtype': HeaderOut
[-4], 'fcontentchecksumtype': HeaderOut
[-3], 'fnumfields': fnumfields
+ 2, 'frawheader': HeaderOut
, 'fextrafields': fextrafields
, 'fextrafieldsize': fextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': fcs
, 'fcontentchecksum': fccs
, 'fhascontents': pyhascontents
, 'fcontentstart': fcontentstart
, 'fcontentend': fcontentend
, 'fcontents': fcontents
};
1027 def ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1028 formatspecs
= FormatSpecsListToDict(formatspecs
);
1029 delimiter
= formatspecs
['format_delimiter'];
1030 fheaderstart
= fp
.tell();
1031 if(formatspecs
['new_style']):
1032 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
1034 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
1035 if(len(HeaderOut
)==0):
1037 fheadsize
= int(HeaderOut
[0], 16);
1038 fnumfields
= int(HeaderOut
[1], 16);
1039 ftype
= int(HeaderOut
[2], 16);
1040 if(re
.findall(r
"^[.|/]", HeaderOut
[3])):
1041 fname
= HeaderOut
[3];
1043 fname
= "./"+HeaderOut
[3];
1044 fbasedir
= os
.path
.dirname(fname
);
1045 flinkname
= HeaderOut
[4];
1046 fsize
= int(HeaderOut
[5], 16);
1047 fatime
= int(HeaderOut
[6], 16);
1048 fmtime
= int(HeaderOut
[7], 16);
1049 fctime
= int(HeaderOut
[8], 16);
1050 fbtime
= int(HeaderOut
[9], 16);
1051 fmode
= int(HeaderOut
[10], 16);
1052 fchmode
= stat
.S_IMODE(fmode
);
1053 ftypemod
= stat
.S_IFMT(fmode
);
1054 fwinattributes
= int(HeaderOut
[11], 16);
1055 fcompression
= HeaderOut
[12];
1056 fcsize
= int(HeaderOut
[13], 16);
1057 fuid
= int(HeaderOut
[14], 16);
1058 funame
= HeaderOut
[15];
1059 fgid
= int(HeaderOut
[16], 16);
1060 fgname
= HeaderOut
[17];
1061 fid
= int(HeaderOut
[18], 16);
1062 finode
= int(HeaderOut
[19], 16);
1063 flinkcount
= int(HeaderOut
[20], 16);
1064 fdev_minor
= int(HeaderOut
[21], 16);
1065 fdev_major
= int(HeaderOut
[22], 16);
1066 frdev_minor
= int(HeaderOut
[23], 16);
1067 frdev_major
= int(HeaderOut
[24], 16);
1068 fseeknextfile
= HeaderOut
[25];
1069 fextrasize
= int(HeaderOut
[26], 16);
1070 fextrafields
= int(HeaderOut
[27], 16);
1071 extrafieldslist
= [];
1073 extraend
= extrastart
+ fextrafields
;
1074 extrafieldslist
= [];
1075 if(extrastart
<extraend
):
1076 extrafieldslist
.append(HeaderOut
[extrastart
]);
1077 extrastart
= extrastart
+ 1;
1078 fheaderchecksumtype
= [extrastart
].lower();
1079 fcontentchecksumtype
= [extrastart
+ 1].lower();
1080 fcs
= HeaderOut
[extrastart
+ 2].lower();
1081 fccs
= HeaderOut
[extrastart
+ 3].lower();
1082 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-4].lower(), True, formatspecs
);
1083 if(fcs
!=newfcs
and not skipchecksum
):
1084 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
1085 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
1087 fhend
= fp
.tell() - 1;
1088 fcontentstart
= fp
.tell();
1089 fcontents
= BytesIO();
1090 pyhascontents
= False;
1091 if(fsize
>0 and not listonly
):
1092 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1093 fcontents
.write(fp
.read(fsize
));
1095 fcontents
.write(fp
.read(fcsize
));
1096 pyhascontents
= True;
1097 elif(fsize
>0 and listonly
):
1098 if(fcompression
=="none" or fcompression
=="" or fcompression
=="atuo"):
1102 pyhascontents
= False;
1103 fcontents
.seek(0, 0);
1104 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
1105 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
1106 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
1107 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
1109 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1112 fcontents
.seek(0, 0);
1114 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
1115 fcontents
.seek(0, 0);
1116 fcontentend
= fp
.tell() - 1;
1117 if(re
.findall(r
"^\+([0-9]+)", fseeknextfile
)):
1118 fseeknextasnum
= int(fseeknextfile
.replace("+", ""));
1119 if(abs(fseeknextasnum
)==0):
1121 fp
.seek(fseeknextasnum
, 1);
1122 elif(re
.findall(r
"^\-([0-9]+)", fseeknextfile
)):
1123 fseeknextasnum
= int(fseeknextfile
);
1124 if(abs(fseeknextasnum
)==0):
1126 fp
.seek(fseeknextasnum
, 1);
1127 elif(re
.findall(r
"^([0-9]+)", fseeknextfile
)):
1128 fseeknextasnum
= int(fseeknextfile
);
1129 if(abs(fseeknextasnum
)==0):
1131 fp
.seek(fseeknextasnum
, 0);
1134 catlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, fseeknextfile
, extrafieldslist
, fheaderchecksumtype
, fcontentchecksumtype
, fcontents
];
1137 def ReadFileDataBySizeWithContent(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1138 formatspecs
= FormatSpecsListToDict(formatspecs
);
1139 delimiter
= formatspecs
['format_delimiter'];
1143 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1146 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1147 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1148 if(not headercheck
and not skipchecksum
):
1149 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1150 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1152 fnumfiles
= int(catheader
[1], 16);
1155 while(countnum
< fnumfiles
):
1156 HeaderOut
= ReadFileHeaderDataBySizeWithContent(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
)
1157 if(len(HeaderOut
)==0):
1159 flist
.append(HeaderOut
);
1160 countnum
= countnum
+ 1;
1163 def ReadFileDataBySizeWithContentToArray(fp
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1164 formatspecs
= FormatSpecsListToDict(formatspecs
);
1165 delimiter
= formatspecs
['format_delimiter'];
1169 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1172 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1173 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1174 if(not headercheck
and not skipchecksum
):
1175 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1176 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1178 catstring
= catheader
[0];
1179 catversion
= re
.findall(r
"([\d]+)", catstring
);
1180 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
1181 fprenumfiles
= catheader
[1];
1182 fnumfiles
= int(fprenumfiles
, 16);
1183 fprechecksumtype
= catheader
[2];
1184 fprechecksum
= catheader
[3];
1185 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': []};
1186 if(seekstart
<0 and seekstart
>fnumfiles
):
1188 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1189 seekend
= fnumfiles
;
1190 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1191 seekend
= fnumfiles
- abs(seekend
);
1194 while(il
< seekstart
):
1195 prefhstart
= fp
.tell();
1196 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1197 if(len(preheaderdata
)==0):
1199 prefsize
= int(preheaderdata
[5], 16);
1200 prefseeknextfile
= preheaderdata
[25];
1201 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
1202 prefcs
= preheaderdata
[-2];
1203 if(prefcs
!=prenewfcs
and not skipchecksum
):
1204 VVerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1205 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1207 valid_archive
= False;
1208 invalid_archive
= True;
1209 prefhend
= fp
.tell() - 1;
1210 prefcontentstart
= fp
.tell();
1211 prefcontents
= BytesIO();
1212 pyhascontents
= False;
1214 prefcontents
.write(fp
.read(prefsize
));
1215 prefcontents
.seek(0, 0);
1216 prenewfccs
= GetFileChecksum(prefcontents
.read(), preheaderdata
[-3].lower(), False, formatspecs
);
1217 prefccs
= preheaderdata
[-1];
1218 pyhascontents
= True;
1219 if(prefccs
!=prenewfccs
and not skipchecksum
):
1220 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1221 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1223 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
1224 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
1225 if(abs(fseeknextasnum
)==0):
1227 fp
.seek(fseeknextasnum
, 1);
1228 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
1229 fseeknextasnum
= int(prefseeknextfile
);
1230 if(abs(fseeknextasnum
)==0):
1232 fp
.seek(fseeknextasnum
, 1);
1233 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
1234 fseeknextasnum
= int(prefseeknextfile
);
1235 if(abs(fseeknextasnum
)==0):
1237 fp
.seek(fseeknextasnum
, 0);
1242 countnum
= seekstart
;
1243 while(countnum
< seekend
):
1244 HeaderOut
= ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1245 if(len(HeaderOut
)==0):
1247 HeaderOut
.update({'fid': realidnum
, 'fidalt': realidnum
});
1248 catlist
['ffilelist'].append(HeaderOut
);
1249 countnum
= countnum
+ 1;
1250 realidnum
= realidnum
+ 1;
1253 def ReadFileDataBySizeWithContentToList(fp
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1254 formatspecs
= FormatSpecsListToDict(formatspecs
);
1255 delimiter
= formatspecs
['format_delimiter'];
1259 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1262 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1263 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1264 if(not headercheck
and not skipchecksum
):
1265 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1266 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1268 catstring
= catheader
[0];
1269 catversion
= re
.findall(r
"([\d]+)", catstring
);
1270 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
1271 fprenumfiles
= catheader
[1];
1272 fnumfiles
= int(fprenumfiles
, 16);
1273 fprechecksumtype
= catheader
[2];
1274 fprechecksum
= catheader
[3];
1276 if(seekstart
<0 and seekstart
>fnumfiles
):
1278 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1279 seekend
= fnumfiles
;
1280 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1281 seekend
= fnumfiles
- abs(seekend
);
1284 while(il
< seekstart
):
1285 prefhstart
= fp
.tell();
1286 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1287 if(len(preheaderdata
)==0):
1289 prefsize
= int(preheaderdata
[5], 16);
1290 prefcompression
= preheaderdata
[12];
1291 prefcsize
= int(preheaderdata
[13], 16);
1292 prefseeknextfile
= HeaderOut
[25];
1293 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
1294 prefcs
= preheaderdata
[-2];
1295 if(prefcs
!=prenewfcs
and not skipchecksum
):
1296 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1297 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1299 valid_archive
= False;
1300 invalid_archive
= True;
1301 prefhend
= fp
.tell() - 1;
1302 prefcontentstart
= fp
.tell();
1304 pyhascontents
= False;
1306 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
1307 prefcontents
= catfp
.read(prefsize
);
1309 prefcontents
= catfp
.read(prefcsize
);
1310 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
1311 prefccs
= preheaderdata
[-1];
1312 pyhascontents
= True;
1313 if(prefccs
!=prenewfccs
and not skipchecksum
):
1314 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1315 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1317 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
1318 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
1319 if(abs(fseeknextasnum
)==0):
1321 catfp
.seek(fseeknextasnum
, 1);
1322 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
1323 fseeknextasnum
= int(prefseeknextfile
);
1324 if(abs(fseeknextasnum
)==0):
1326 catfp
.seek(fseeknextasnum
, 1);
1327 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
1328 fseeknextasnum
= int(prefseeknextfile
);
1329 if(abs(fseeknextasnum
)==0):
1331 catfp
.seek(fseeknextasnum
, 0);
1336 countnum
= seekstart
;
1337 while(countnum
< seekend
):
1338 HeaderOut
= ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1339 if(len(HeaderOut
)==0):
1341 catlist
.append(HeaderOut
);
1342 countnum
= countnum
+ 1;
1343 realidnum
= realidnum
+ 1;
1346 def ReadInFileBySizeWithContentToArray(infile
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1347 formatspecs
= FormatSpecsListToDict(formatspecs
);
1348 delimiter
= formatspecs
['format_delimiter'];
1349 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1352 fp
= UncompressArchiveFile(fp
, formatspecs
);
1353 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1354 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1361 if(hasattr(sys
.stdin
, "buffer")):
1362 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1364 shutil
.copyfileobj(sys
.stdin
, fp
);
1366 fp
= UncompressArchiveFile(fp
, formatspecs
);
1370 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1371 fp
= download_file_from_internet_file(infile
);
1372 fp
= UncompressArchiveFile(fp
, formatspecs
);
1378 infile
= RemoveWindowsPath(infile
);
1379 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1380 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1382 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1383 if(not compresscheck
):
1384 fextname
= os
.path
.splitext(infile
)[1];
1385 if(fextname
==".gz"):
1386 compresscheck
= "gzip";
1387 elif(fextname
==".bz2"):
1388 compresscheck
= "bzip2";
1389 elif(fextname
==".zst"):
1390 compresscheck
= "zstd";
1391 elif(fextname
==".lz4" or fextname
==".clz4"):
1392 compresscheck
= "lz4";
1393 elif(fextname
==".lzo" or fextname
==".lzop"):
1394 compresscheck
= "lzo";
1395 elif(fextname
==".lzma" or fextname
==".xz"):
1396 compresscheck
= "lzma";
1399 if(not compresscheck
):
1401 fp
= UncompressFile(infile
, formatspecs
, "rb");
1402 return ReadFileDataBySizeWithContentToArray(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1404 def ReadInFileBySizeWithContentToList(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1405 formatspecs
= FormatSpecsListToDict(formatspecs
);
1406 delimiter
= formatspecs
['format_delimiter'];
1407 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1410 fp
= UncompressArchiveFile(fp
, formatspecs
);
1411 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1412 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1419 if(hasattr(sys
.stdin
, "buffer")):
1420 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1422 shutil
.copyfileobj(sys
.stdin
, fp
);
1424 fp
= UncompressArchiveFile(fp
, formatspecs
);
1428 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1429 fp
= download_file_from_internet_file(infile
);
1430 fp
= UncompressArchiveFile(fp
, formatspecs
);
1436 infile
= RemoveWindowsPath(infile
);
1437 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1438 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1440 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1441 if(not compresscheck
):
1442 fextname
= os
.path
.splitext(infile
)[1];
1443 if(fextname
==".gz"):
1444 compresscheck
= "gzip";
1445 elif(fextname
==".bz2"):
1446 compresscheck
= "bzip2";
1447 elif(fextname
==".zst"):
1448 compresscheck
= "zstd";
1449 elif(fextname
==".lz4" or fextname
==".clz4"):
1450 compresscheck
= "lz4";
1451 elif(fextname
==".lzo" or fextname
==".lzop"):
1452 compresscheck
= "lzo";
1453 elif(fextname
==".lzma" or fextname
==".xz"):
1454 compresscheck
= "lzma";
1457 if(not compresscheck
):
1459 fp
= UncompressFile(infile
, formatspecs
, "rb");
1460 return ReadFileDataBySizeWithContentToList(fp
, seekstart
, seekend
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1462 def AppendNullByte(indata
, delimiter
=__file_format_dict__
['format_delimiter']):
1463 outdata
= str(indata
) + delimiter
;
1466 def AppendNullBytes(indata
=[], delimiter
=__file_format_dict__
['format_delimiter']):
1471 outdata
= outdata
+ AppendNullByte(indata
[inum
], delimiter
);
1475 def AppendFileHeader(fp
, numfiles
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1476 formatspecs
= FormatSpecsListToDict(formatspecs
);
1477 delimiter
= formatspecs
['format_delimiter'];
1478 catver
= formatspecs
['format_ver'];
1479 fileheaderver
= str(int(catver
.replace(".", "")));
1480 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
1481 fp
.write(fileheader
.encode('UTF-8'));
1482 fnumfiles
= format(int(numfiles
), 'x').lower();
1483 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
['format_delimiter']);
1484 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
1485 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
1486 fp
.write(fnumfilesa
.encode('UTF-8'));
1489 os
.fsync(fp
.fileno());
1490 except io
.UnsupportedOperation
:
1492 except AttributeError:
1494 except OSError as e
:
1498 def MakeEmptyFilePointer(fp
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1499 formatspecs
= FormatSpecsListToDict(formatspecs
);
1500 AppendFileHeader(fp
, 0, checksumtype
, formatspecs
);
1503 def MakeEmptyFile(outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", formatspecs
=__file_format_dict__
, returnfp
=False):
1504 formatspecs
= FormatSpecsListToDict(formatspecs
);
1505 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1506 if(os
.path
.exists(outfile
)):
1509 except OSError as e
:
1513 catfpfp
= BytesIO();
1514 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1516 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1519 fbasename
= os
.path
.splitext(outfile
)[0];
1520 fextname
= os
.path
.splitext(outfile
)[1];
1521 catfp
= CompressOpenFile(outfile
, True, compressionlevel
);
1522 catfp
= AppendFileHeader(catfp
, 0, checksumtype
, formatspecs
);
1523 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1524 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1527 os
.fsync(catfp
.fileno());
1528 except io
.UnsupportedOperation
:
1530 except AttributeError:
1532 except OSError as e
:
1536 if(hasattr(sys
.stdout
, "buffer")):
1537 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1539 shutil
.copyfileobj(catfp
, sys
.stdout
);
1540 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1541 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1543 upload_file_to_internet_file(catfp
, outfile
);
1551 def AppendFileHeaderWithContent(fp
, filevalues
=[], extradata
=[], filecontent
="", checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1552 formatspecs
= FormatSpecsListToDict(formatspecs
);
1553 extrafields
= format(len(extradata
), 'x').lower();
1554 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
1555 if(len(extradata
)>0):
1556 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1557 extrasizelen
= format(len(extrasizestr
), 'x').lower();
1558 catoutlen
= len(filevalues
) + len(extradata
) + 5;
1559 catoutlenhex
= format(catoutlen
, 'x').lower();
1560 catoutlist
= filevalues
;
1561 catoutlist
.insert(0, catoutlenhex
);
1562 catoutlist
.append(extrasizelen
);
1563 catoutlist
.append(extrafields
);
1564 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
1565 if(len(extradata
)>0):
1566 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1567 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
1568 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1569 catfilecontentcshex
= GetFileChecksum(filecontent
, checksumtype
, False, formatspecs
);
1570 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1571 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
1572 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
1573 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1574 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1575 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
1576 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
1577 catfileout
= catfileoutstrecd
+ filecontent
+ nullstrecd
;
1578 fp
.write(catfileout
);
1581 os
.fsync(fp
.fileno());
1582 except io
.UnsupportedOperation
:
1584 except AttributeError:
1586 except OSError as e
:
1590 def AppendFilesWithContent(infiles
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1591 formatspecs
= FormatSpecsListToDict(formatspecs
);
1592 advancedlist
= formatspecs
['use_advanced_list'];
1593 altinode
= formatspecs
['use_alt_inode'];
1595 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1597 for line
in sys
.stdin
:
1598 infilelist
.append(line
.strip());
1599 infilelist
= list(filter(None, infilelist
));
1600 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
1601 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
1603 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
1604 for line
in finfile
:
1605 infilelist
.append(line
.strip());
1606 infilelist
= list(filter(None, infilelist
));
1608 if(isinstance(infiles
, (list, tuple, ))):
1609 infilelist
= list(filter(None, infiles
));
1610 elif(isinstance(infiles
, (str, ))):
1611 infilelist
= list(filter(None, [infiles
]));
1613 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
1615 GetDirList
= ListDir(infilelist
, followlink
, False);
1623 inodetocatinode
= {};
1624 numfiles
= int(len(GetDirList
));
1625 fnumfiles
= format(numfiles
, 'x').lower();
1626 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1627 for curfname
in GetDirList
:
1628 if(re
.findall(r
"^[.|/]", curfname
)):
1631 fname
= "./"+curfname
;
1633 VerbosePrintOut(fname
);
1634 if(not followlink
or followlink
is None):
1635 fstatinfo
= os
.lstat(fname
);
1637 fstatinfo
= os
.stat(fname
);
1638 fpremode
= fstatinfo
.st_mode
;
1639 finode
= fstatinfo
.st_ino
;
1640 flinkcount
= fstatinfo
.st_nlink
;
1642 if(stat
.S_ISREG(fpremode
)):
1644 elif(stat
.S_ISLNK(fpremode
)):
1646 elif(stat
.S_ISCHR(fpremode
)):
1648 elif(stat
.S_ISBLK(fpremode
)):
1650 elif(stat
.S_ISDIR(fpremode
)):
1652 elif(stat
.S_ISFIFO(fpremode
)):
1654 elif(stat
.S_ISSOCK(fpremode
)):
1656 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
1658 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
1660 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
1665 fcurfid
= format(int(curfid
), 'x').lower();
1666 if(not followlink
and finode
!=0):
1668 if(finode
in inodelist
):
1670 flinkname
= inodetofile
[finode
];
1672 fcurinode
= format(int(finode
), 'x').lower();
1674 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
1675 if(finode
not in inodelist
):
1676 inodelist
.append(finode
);
1677 inodetofile
.update({finode
: fname
});
1678 inodetocatinode
.update({finode
: curinode
});
1680 fcurinode
= format(int(finode
), 'x').lower();
1682 fcurinode
= format(int(curinode
), 'x').lower();
1683 curinode
= curinode
+ 1;
1685 fcurinode
= format(int(curinode
), 'x').lower();
1686 curinode
= curinode
+ 1;
1687 curfid
= curfid
+ 1;
1689 flinkname
= os
.readlink(fname
);
1690 fdev
= fstatinfo
.st_dev
;
1691 getfdev
= GetDevMajorMinor(fdev
);
1692 fdev_minor
= getfdev
[0];
1693 fdev_major
= getfdev
[1];
1694 frdev
= fstatinfo
.st_dev
;
1695 if(hasattr(fstatinfo
, "st_rdev")):
1696 frdev
= fstatinfo
.st_rdev
;
1698 frdev
= fstatinfo
.st_dev
;
1699 getfrdev
= GetDevMajorMinor(frdev
);
1700 frdev_minor
= getfrdev
[0];
1701 frdev_major
= getfrdev
[1];
1702 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
1703 fsize
= format(int("0"), 'x').lower();
1704 elif(ftype
==0 or ftype
==7):
1705 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
1707 fsize
= format(int(fstatinfo
.st_size
)).lower();
1708 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
1709 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
1710 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1711 if(hasattr(fstatinfo
, "st_birthtime")):
1712 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
1714 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1715 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
1716 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
1717 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
1718 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
1719 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
1724 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
1725 funame
= userinfo
.pw_name
;
1734 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
1735 fgname
= groupinfo
.gr_name
;
1740 fdev_minor
= format(int(fdev_minor
), 'x').lower();
1741 fdev_major
= format(int(fdev_major
), 'x').lower();
1742 frdev_minor
= format(int(frdev_minor
), 'x').lower();
1743 frdev_major
= format(int(frdev_major
), 'x').lower();
1744 finode
= format(int(finode
), 'x').lower();
1745 flinkcount
= format(int(flinkcount
), 'x').lower();
1746 if(hasattr(fstatinfo
, "st_file_attributes")):
1747 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
1749 fwinattributes
= format(int(0), 'x').lower();
1751 fcsize
= format(int(0), 'x').lower();
1752 fcontents
= BytesIO();
1754 if(ftype
==0 or ftype
==7):
1755 with
open(fname
, "rb") as fpc
:
1756 shutil
.copyfileobj(fpc
, fcontents
);
1757 if(not compresswholefile
):
1758 fcontents
.seek(0, 2);
1759 ucfsize
= fcontents
.tell();
1760 fcontents
.seek(0, 0);
1761 if(compression
=="auto"):
1762 ilsize
= len(compressionlistalt
);
1765 while(ilmin
< ilsize
):
1766 cfcontents
= BytesIO();
1767 shutil
.copyfileobj(fcontents
, cfcontents
);
1768 fcontents
.seek(0, 0);
1769 cfcontents
.seek(0, 0);
1770 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1772 cfcontents
.seek(0, 2);
1773 ilcsize
.append(cfcontents
.tell());
1777 ilcsize
.append(sys
.maxint
);
1778 except AttributeError:
1779 ilcsize
.append(sys
.maxsize
);
1781 ilcmin
= ilcsize
.index(min(ilcsize
));
1782 compression
= compressionlistalt
[ilcmin
];
1783 fcontents
.seek(0, 0);
1784 cfcontents
= BytesIO();
1785 shutil
.copyfileobj(fcontents
, cfcontents
);
1786 cfcontents
.seek(0, 0);
1787 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1788 cfcontents
.seek(0, 2);
1789 cfsize
= cfcontents
.tell();
1790 if(ucfsize
> cfsize
):
1791 fcsize
= format(int(cfsize
), 'x').lower();
1792 fcompression
= compression
;
1794 fcontents
= cfcontents
;
1795 if(followlink
and (ftype
==1 or ftype
==2)):
1796 flstatinfo
= os
.stat(flinkname
);
1797 with
open(flinkname
, "rb") as fpc
:
1798 shutil
.copyfileobj(fpc
, fcontents
);
1799 if(not compresswholefile
):
1800 fcontents
.seek(0, 2);
1801 ucfsize
= fcontents
.tell();
1802 fcontents
.seek(0, 0);
1803 if(compression
=="auto"):
1804 ilsize
= len(compressionlistalt
);
1807 while(ilmin
< ilsize
):
1808 cfcontents
= BytesIO();
1809 shutil
.copyfileobj(fcontents
, cfcontents
);
1810 fcontents
.seek(0, 0);
1811 cfcontents
.seek(0, 0);
1812 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1814 cfcontents
.seek(0, 2);
1815 ilcsize
.append(cfcontents
.tell());
1819 ilcsize
.append(sys
.maxint
);
1820 except AttributeError:
1821 ilcsize
.append(sys
.maxsize
);
1823 ilcmin
= ilcsize
.index(min(ilcsize
));
1824 compression
= compressionlistalt
[ilcmin
];
1825 fcontents
.seek(0, 0);
1826 cfcontents
= BytesIO();
1827 shutil
.copyfileobj(fcontents
, cfcontents
);
1828 cfcontents
.seek(0, 0);
1829 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1830 cfcontents
.seek(0, 2);
1831 cfsize
= cfcontents
.tell();
1832 if(ucfsize
> cfsize
):
1833 fcsize
= format(int(cfsize
), 'x').lower();
1834 fcompression
= compression
;
1836 fcontents
= cfcontents
;
1837 if(fcompression
=="none"):
1839 fcontents
.seek(0, 0);
1840 ftypehex
= format(ftype
, 'x').lower();
1841 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
1842 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1844 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1848 def AppendListsWithContent(inlist
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1849 formatspecs
= FormatSpecsListToDict(formatspecs
);
1851 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1852 GetDirList
= inlist
;
1860 inodetocatinode
= {};
1861 numfiles
= int(len(GetDirList
));
1862 fnumfiles
= format(numfiles
, 'x').lower();
1863 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1864 for curfname
in GetDirList
:
1865 ftype
= format(curfname
[0], 'x').lower();
1866 if(re
.findall(r
"^[.|/]", curfname
[1])):
1867 fname
= curfname
[1];
1869 fname
= "./"+curfname
[1];
1870 fbasedir
= os
.path
.dirname(fname
);
1871 flinkname
= curfname
[2];
1872 fsize
= format(curfname
[3], 'x').lower();
1873 fatime
= format(curfname
[4], 'x').lower();
1874 fmtime
= format(curfname
[5], 'x').lower();
1875 fctime
= format(curfname
[6], 'x').lower();
1876 fbtime
= format(curfname
[7], 'x').lower();
1877 fmode
= format(curfname
[8], 'x').lower();
1878 fwinattributes
= format(curfname
[9], 'x').lower();
1879 fcompression
= curfname
[10];
1880 fcsize
= format(curfname
[11], 'x').lower();
1881 fuid
= format(curfname
[12], 'x').lower();
1882 funame
= curfname
[13];
1883 fgid
= format(curfname
[14], 'x').lower();
1884 fgname
= curfname
[15];
1885 fid
= format(curfname
[16], 'x').lower();
1886 finode
= format(curfname
[17], 'x').lower();
1887 flinkcount
= format(curfname
[18], 'x').lower();
1888 fdev_minor
= format(curfname
[19], 'x').lower();
1889 fdev_major
= format(curfname
[20], 'x').lower();
1890 frdev_minor
= format(curfname
[21], 'x').lower();
1891 frdev_major
= format(curfname
[22], 'x').lower();
1892 fseeknextfile
= curfname
[23];
1893 extradata
= curfname
[24];
1894 fheaderchecksumtype
= curfname
[25];
1895 fcontentchecksumtype
= curfname
[26];
1896 fcontents
= curfname
[27];
1897 catoutlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, fseeknextfile
];
1898 fcontents
.seek(0, 0);
1899 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1901 fp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1904 def AppendInFileWithContent(infile
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1905 formatspecs
= FormatSpecsListToDict(formatspecs
);
1906 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, True, False, formatspecs
);
1907 return AppendListsWithContent(inlist
, fp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1909 def AppendFilesWithContentToOutFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1910 formatspecs
= FormatSpecsListToDict(formatspecs
);
1911 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1912 if(os
.path
.exists(outfile
)):
1915 except OSError as e
:
1919 catfpfp
= BytesIO();
1920 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1922 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1925 fbasename
= os
.path
.splitext(outfile
)[0];
1926 fextname
= os
.path
.splitext(outfile
)[1];
1927 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1928 catfp
= AppendFilesWithContent(infiles
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1929 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1930 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1933 os
.fsync(catfp
.fileno());
1934 except io
.UnsupportedOperation
:
1936 except AttributeError:
1938 except OSError as e
:
1942 if(hasattr(sys
.stdout
, "buffer")):
1943 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1945 shutil
.copyfileobj(catfp
, sys
.stdout
);
1946 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1947 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1949 upload_file_to_internet_file(catfp
, outfile
);
1957 def AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1958 formatspecs
= FormatSpecsListToDict(formatspecs
);
1959 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1960 if(os
.path
.exists(outfile
)):
1963 except OSError as e
:
1967 catfpfp
= BytesIO();
1968 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1970 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1973 fbasename
= os
.path
.splitext(outfile
)[0];
1974 fextname
= os
.path
.splitext(outfile
)[1];
1975 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1976 catfp
= AppendListsWithContent(inlist
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1977 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1978 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1981 os
.fsync(catfp
.fileno());
1982 except io
.UnsupportedOperation
:
1984 except AttributeError:
1986 except OSError as e
:
1990 if(hasattr(sys
.stdout
, "buffer")):
1991 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1993 shutil
.copyfileobj(catfp
, sys
.stdout
);
1994 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1995 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1997 upload_file_to_internet_file(catfp
, outfile
);
2005 def AppendInFileWithContentToOutFile(infile
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2006 formatspecs
= FormatSpecsListToDict(formatspecs
);
2007 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, True, False, formatspecs
);
2008 return AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
, returnfp
);
2010 def PrintPermissionString(fchmode
, ftype
):
2011 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2013 for fmodval
in str(oct(fchmode
))[-3:]:
2014 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
2015 if(ftype
==0 or ftype
==7):
2016 permissionstr
= "-" + permissionstr
;
2018 permissionstr
= "h" + permissionstr
;
2020 permissionstr
= "l" + permissionstr
;
2022 permissionstr
= "c" + permissionstr
;
2024 permissionstr
= "b" + permissionstr
;
2026 permissionstr
= "d" + permissionstr
;
2028 permissionstr
= "f" + permissionstr
;
2030 permissionstr
= "D" + permissionstr
;
2032 permissionstr
= "p" + permissionstr
;
2034 permissionstr
= "w" + permissionstr
;
2036 permissionoutstr
= stat
.filemode(fchmode
);
2037 except AttributeError:
2038 permissionoutstr
= permissionstr
;
2040 permissionoutstr
= permissionstr
;
2041 return permissionoutstr
;
2043 def PrintPermissionStringAlt(fchmode
, ftype
):
2045 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2046 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2048 # Translate file mode into permission string
2049 permissionstr
= ''.join([permissions
[i
] for i
in str(oct(fchmode
))[-3:]]);
2050 # Append file type indicator
2052 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2053 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2055 file_type
= type_indicators
.get(ftype
, '-');
2056 permissionstr
= file_type
+ permissionstr
;
2058 permissionoutstr
= stat
.filemode(fchmode
);
2059 except AttributeError:
2060 permissionoutstr
= permissionstr
;
2061 return permissionoutstr
;
2063 def CheckCompressionType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2064 formatspecs
= FormatSpecsListToDict(formatspecs
);
2065 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2069 catfp
= open(infile
, "rb");
2070 except FileNotFoundError
:
2073 prefp
= catfp
.read(2);
2075 if(prefp
==binascii
.unhexlify("1f8b")):
2078 prefp
= catfp
.read(3);
2079 if(prefp
==binascii
.unhexlify("425a68")):
2081 if(prefp
==binascii
.unhexlify("5d0000")):
2084 prefp
= catfp
.read(4);
2085 if(prefp
==binascii
.unhexlify("28b52ffd")):
2087 if(prefp
==binascii
.unhexlify("04224d18")):
2089 if(prefp
==binascii
.unhexlify("504B0304")):
2090 filetype
= "zipfile";
2092 prefp
= catfp
.read(5);
2093 if(prefp
==binascii
.unhexlify("7573746172")):
2094 filetype
= "tarfile";
2096 prefp
= catfp
.read(6);
2097 if(prefp
==binascii
.unhexlify("fd377a585a00")):
2099 if(prefp
==binascii
.unhexlify("377abcaf271c")):
2100 filetype
= "7zipfile";
2102 prefp
= catfp
.read(7);
2103 if(prefp
==binascii
.unhexlify("526172211a0700")):
2104 filetype
= "rarfile";
2105 if(prefp
==binascii
.unhexlify("43617446696c65")):
2106 filetype
= "catfile";
2108 prefp
= catfp
.read(8);
2109 if(prefp
==binascii
.unhexlify("526172211a070100")):
2110 filetype
= "rarfile";
2112 prefp
= catfp
.read(formatspecs
['format_len']);
2113 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2114 filetype
= formatspecs
['format_lower'];
2116 prefp
= catfp
.read(9);
2117 if(prefp
==binascii
.unhexlify("894c5a4f000d0a1a0a")):
2120 prefp
= catfp
.read(10);
2121 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2122 filetype
= "tarfile";
2124 if(filetype
=="gzip" or filetype
=="bzip2" or filetype
=="lzma" or filetype
=="zstd" or filetype
=="lz4"):
2125 if(TarFileCheck(catfp
)):
2126 filetype
= "tarfile";
2128 if(TarFileCheck(catfp
)):
2129 filetype
= "tarfile";
2130 elif(zipfile
.is_zipfile(catfp
)):
2131 filetype
= "zipfile";
2132 elif(rarfile
.is_rarfile(catfp
) or rarfile
.is_rarfile_sfx(catfp
)):
2133 filetype
= "rarile";
2141 def CheckCompressionTypeFromString(instring
, formatspecs
=__file_format_dict__
, closefp
=True):
2142 formatspecs
= FormatSpecsListToDict(formatspecs
);
2144 instringsfile
= BytesIO(instring
);
2146 instringsfile
= BytesIO(instring
.encode("UTF-8"));
2147 return CheckCompressionType(instringsfile
, formatspecs
, closefp
);
2149 def GetCompressionMimeType(infile
, formatspecs
=__file_format_dict__
):
2150 formatspecs
= FormatSpecsListToDict(formatspecs
);
2151 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2152 if(compresscheck
=="gzip" or compresscheck
=="gz"):
2153 return archivefile_gzip_mimetype
;
2154 if(compresscheck
=="bzip2" or compresscheck
=="bz2"):
2155 return archivefile_bzip2_mimetype
;
2156 if(compresscheck
=="zstd" or compresscheck
=="zstandard"):
2157 return archivefile_zstandard_mimetype
;
2158 if(compresscheck
=="lz4"):
2159 return archivefile_lz4_mimetype
;
2160 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2161 return archivefile_lzop_mimetype
;
2162 if(compresscheck
=="lzma"):
2163 return archivefile_lzma_mimetype
;
2164 if(compresscheck
=="xz"):
2165 return archivefile_xz_mimetype
;
2166 if(compresscheck
=="catfile" or compresscheck
=="cat" or compresscheck
==formatspecs
['format_lower']):
2167 return archivefile_cat_mimetype
;
2168 if(not compresscheck
):
2172 def UncompressArchiveFile(fp
, formatspecs
=__file_format_dict__
):
2173 formatspecs
= FormatSpecsListToDict(formatspecs
);
2174 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2176 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2177 if(compresscheck
=="gzip"):
2182 catfp
= gzip
.GzipFile(fileobj
=fp
, mode
="rb");
2183 if(compresscheck
=="bzip2"):
2189 catfp
.write(bz2
.decompress(fp
.read()));
2190 if(compresscheck
=="zstd"):
2196 catfp
.write(zstandard
.decompress(fp
.read()));
2197 if(compresscheck
=="lz4"):
2203 catfp
.write(lz4
.frame
.decompress(fp
.read()));
2204 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2210 catfp
.write(lzo
.decompress(fp
.read()));
2211 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2216 from backports
import lzma
2220 catfp
.write(lzma
.decompress(fp
.read()));
2221 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2223 if(not compresscheck
):
2228 from backports
import lzma
2232 with fp
as fpcontent
:
2234 catfp
.write(lzma
.decompress(fp
.read()));
2235 except lzma
.LZMAError
:
2237 if(compresscheck
!="catfile" or compresscheck
!=formatspecs
['format_lower']):
2241 create_alias_function("Uncompress", __file_format_name__
, "", UncompressArchiveFile
);
2243 def UncompressFile(infile
, formatspecs
=__file_format_dict__
, mode
="rb"):
2244 formatspecs
= FormatSpecsListToDict(formatspecs
);
2245 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2246 if(sys
.version_info
[0]==2 and compresscheck
):
2252 if(compresscheck
=="gzip"):
2258 filefp
= gzip
.open(infile
, mode
, encoding
="UTF-8");
2259 except (ValueError, TypeError) as e
:
2260 filefp
= gzip
.open(infile
, mode
);
2261 if(compresscheck
=="bzip2"):
2267 filefp
= bz2
.open(infile
, mode
, encoding
="UTF-8");
2268 except (ValueError, TypeError) as e
:
2269 filefp
= bz2
.open(infile
, mode
);
2270 if(compresscheck
=="zstd"):
2276 filefp
= zstandard
.open(infile
, mode
, encoding
="UTF-8");
2277 except (ValueError, TypeError) as e
:
2278 filefp
= zstandard
.open(infile
, mode
);
2279 if(compresscheck
=="lz4"):
2285 filefp
= lz4
.frame
.open(infile
, mode
, encoding
="UTF-8");
2286 except (ValueError, TypeError) as e
:
2287 filefp
= lz4
.frame
.open(infile
, mode
);
2288 if(compresscheck
=="lzo"):
2294 filefp
= lzo
.open(infile
, mode
, encoding
="UTF-8");
2295 except (ValueError, TypeError) as e
:
2296 filefp
= lzo
.open(infile
, mode
);
2297 if(compresscheck
=="lzma"):
2302 from backports
import lzma
2306 filefp
= lzma
.open(infile
, mode
, encoding
="UTF-8");
2307 except (ValueError, TypeError) as e
:
2308 filefp
= lzma
.open(infile
, mode
);
2309 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2311 filefp
= open(infile
, mode
, encoding
="UTF-8");
2312 except (ValueError, TypeError) as e
:
2313 filefp
= open(infile
, mode
);
2314 if(not compresscheck
):
2316 filefp
= open(infile
, mode
, encoding
="UTF-8");
2317 except (ValueError, TypeError) as e
:
2318 filefp
= open(infile
, mode
);
2319 except FileNotFoundError
:
2323 def UncompressString(infile
):
2324 compresscheck
= CheckCompressionTypeFromString(infile
, formatspecs
, False);
2325 if(compresscheck
=="gzip"):
2330 fileuz
= gzip
.decompress(infile
);
2331 if(compresscheck
=="bzip2"):
2336 fileuz
= bz2
.decompress(infile
);
2337 if(compresscheck
=="zstd"):
2342 fileuz
= zstandard
.decompress(infile
);
2343 if(compresscheck
=="lz4"):
2348 fileuz
= lz4
.frame
.decompress(infile
);
2349 if(compresscheck
=="lzo"):
2354 fileuz
= lzo
.decompress(infile
);
2355 if(compresscheck
=="lzma"):
2360 from backports
import lzma
2363 fileuz
= lzma
.decompress(infile
);
2364 if(not compresscheck
):
2366 if(hasattr(fileuz
, 'decode')):
2367 fileuz
= fileuz
.decode("UTF-8");
2370 def UncompressStringAlt(infile
):
2371 filefp
= StringIO();
2372 outstring
= UncompressString(infile
);
2373 filefp
.write(outstring
);
2377 def CheckCompressionSubType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2378 formatspecs
= FormatSpecsListToDict(formatspecs
);
2379 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2380 if(not compresscheck
):
2381 fextname
= os
.path
.splitext(infile
)[1];
2382 if(fextname
==".gz"):
2383 compresscheck
= "gzip";
2384 elif(fextname
==".bz2"):
2385 compresscheck
= "bzip2";
2386 elif(fextname
==".zst"):
2387 compresscheck
= "zstd";
2388 elif(fextname
==".lz4"):
2389 compresscheck
= "lz4";
2390 elif(fextname
==".lzo" or fextname
==".lzop"):
2391 compresscheck
= "lzo";
2392 elif(fextname
==".lzma" or fextname
==".xz"):
2393 compresscheck
= "lzma";
2396 if(compresscheck
=="gzip" or compresscheck
=="bzip2" or compresscheck
=="lzma" or compresscheck
=="zstd" or compresscheck
=="lz4"):
2397 if(TarFileCheck(infile
)):
2398 filetype
= "tarfile";
2399 if(not compresscheck
):
2400 if(TarFileCheck(infile
)):
2402 elif(zipfile
.is_zipfile(infile
)):
2404 elif(rarfile
.is_rarfile(infile
) or rarfile
.is_rarfile_sfx(infile
)):
2409 if(compresscheck
=="catfile"):
2411 if(compresscheck
==formatspecs
['format_lower']):
2412 return formatspecs
['format_lower'];
2413 if(compresscheck
=="tarfile"):
2415 if(compresscheck
=="zipfile"):
2417 if(rarfile_support
and compresscheck
=="rarfile"):
2419 if(py7zr_support
and compresscheck
=="7zipfile"):
2421 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2422 catfp
= UncompressArchiveFile(infile
, formatspecs
['format_lower']);
2425 if(compresscheck
=="gzip"):
2430 catfp
= gzip
.GzipFile(infile
, "rb");
2431 if(compresscheck
=="bzip2"):
2436 catfp
= bz2
.BZ2File(infile
, "rb");
2437 if(compresscheck
=="lz4"):
2442 catfp
= lz4
.frame
.open(infile
, "rb");
2443 if(compresscheck
=="zstd"):
2448 catfp
= zstandard
.open(infile
, "rb");
2449 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2454 from backports
import lzma
2457 catfp
= lzma
.open(infile
, "rb");
2458 except FileNotFoundError
:
2461 prefp
= catfp
.read(5);
2462 if(prefp
==binascii
.unhexlify("7573746172")):
2463 filetype
= "tarfile";
2465 prefp
= catfp
.read(7);
2466 if(prefp
==binascii
.unhexlify("43617446696c65")):
2467 filetype
= "catfile";
2469 prefp
= catfp
.read(formatspecs
['format_len']);
2470 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2471 filetype
= formatspecs
['format_lower'];
2473 prefp
= catfp
.read(10);
2474 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2475 filetype
= "tarfile";
2481 def GZipCompress(data
, compresslevel
=9):
2486 tmpfp
= tempfile
.NamedTemporaryFile("wb", delete
=False);
2488 tmpfp
= gzip
.GzipFile(tmpfp
.name
, mode
="wb", compresslevel
=compresslevel
);
2492 catfp
= open(tmpfp
.name
, "rb");
2493 except FileNotFoundError
:
2495 catdata
= catfp
.read();
2499 def CompressArchiveFile(fp
, compression
="auto", compressionlevel
=None, formatspecs
=__file_format_dict__
):
2500 formatspecs
= FormatSpecsListToDict(formatspecs
);
2501 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2504 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2505 compression
= "auto";
2506 if(compression
not in compressionlist
and compression
is None):
2507 compression
= "auto";
2508 if(compression
=="gzip"):
2514 if(compressionlevel
is None):
2515 compressionlevel
= 9;
2517 compressionlevel
= int(compressionlevel
);
2518 catfp
.write(gzip
.compress(fp
.read(), compresslevel
=compressionlevel
));
2519 if(compression
=="bzip2"):
2525 if(compressionlevel
is None):
2526 compressionlevel
= 9;
2528 compressionlevel
= int(compressionlevel
);
2529 catfp
.write(bz2
.compress(fp
.read(), compresslevel
=compressionlevel
));
2530 if(compression
=="lz4"):
2536 if(compressionlevel
is None):
2537 compressionlevel
= 9;
2539 compressionlevel
= int(compressionlevel
);
2540 catfp
.write(lz4
.frame
.compress(fp
.read(), compression_level
=compressionlevel
));
2541 if(compression
=="lzo" or compression
=="lzop"):
2547 if(compressionlevel
is None):
2548 compressionlevel
= 9;
2550 compressionlevel
= int(compressionlevel
);
2551 catfp
.write(lzo
.compress(fp
.read(), compresslevel
=compressionlevel
));
2552 if(compression
=="zstd"):
2558 if(compressionlevel
is None):
2559 compressionlevel
= 10;
2561 compressionlevel
= int(compressionlevel
);
2562 catfp
.write(zstandard
.compress(fp
.read(), level
=compressionlevel
));
2563 if(compression
=="lzma"):
2568 from backports
import lzma
2572 if(compressionlevel
is None):
2573 compressionlevel
= 9;
2575 compressionlevel
= int(compressionlevel
);
2576 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]));
2577 if(compression
=="xz"):
2582 from backports
import lzma
2586 if(compressionlevel
is None):
2587 compressionlevel
= 9;
2589 compressionlevel
= int(compressionlevel
);
2590 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]));
2591 if(compression
=="auto" or compression
is None):
2596 create_alias_function("Compress", __file_format_name__
, "", CompressArchiveFile
);
2598 def CompressOpenFile(outfile
, compressionenable
=True, compressionlevel
=None):
2599 if(outfile
is None):
2601 fbasename
= os
.path
.splitext(outfile
)[0];
2602 fextname
= os
.path
.splitext(outfile
)[1];
2603 if(compressionlevel
is None and fextname
!=".zst"):
2604 compressionlevel
= 9;
2605 elif(compressionlevel
is None and fextname
==".zst"):
2606 compressionlevel
= 10;
2608 compressionlevel
= int(compressionlevel
);
2609 if(sys
.version_info
[0]==2):
2614 if(fextname
not in outextlistwd
or not compressionenable
):
2616 outfp
= open(outfile
, "wb", encoding
="UTF-8");
2617 except (ValueError, TypeError) as e
:
2618 outfp
= open(outfile
, "wb");
2619 elif(fextname
==".gz"):
2625 outfp
= gzip
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2626 except (ValueError, TypeError) as e
:
2627 outfp
= gzip
.open(outfile
, mode
, compressionlevel
);
2628 elif(fextname
==".bz2"):
2634 outfp
= bz2
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2635 except (ValueError, TypeError) as e
:
2636 outfp
= bz2
.open(outfile
, mode
, compressionlevel
);
2637 elif(fextname
==".zst"):
2643 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
), encoding
="UTF-8");
2644 except (ValueError, TypeError) as e
:
2645 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
));
2646 elif(fextname
==".xz"):
2651 from backports
import lzma
2655 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}], encoding
="UTF-8");
2656 except (ValueError, TypeError) as e
:
2657 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]);
2658 elif(fextname
==".lz4"):
2664 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
, encoding
="UTF-8");
2665 except (ValueError, TypeError) as e
:
2666 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
);
2667 elif(fextname
==".lzo"):
2673 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
, encoding
="UTF-8");
2674 except (ValueError, TypeError) as e
:
2675 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
);
2676 elif(fextname
==".lzma"):
2681 from backports
import lzma
2685 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}], encoding
="UTF-8");
2686 except (ValueError, TypeError) as e
:
2687 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]);
2688 except FileNotFoundError
:
2692 def GetDevMajorMinor(fdev
):
2694 if(hasattr(os
, "minor")):
2695 retdev
.append(os
.minor(fdev
));
2698 if(hasattr(os
, "major")):
2699 retdev
.append(os
.major(fdev
));
2704 def CheckSumSupport(checkfor
, guaranteed
=True):
2707 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2708 except AttributeError:
2709 hash_list
= sorted(list(hashlib
.algorithms
));
2712 hash_list
= sorted(list(hashlib
.algorithms_available
));
2713 except AttributeError:
2714 hash_list
= sorted(list(hashlib
.algorithms
));
2715 checklistout
= sorted(hash_list
+ ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2716 if(checkfor
in checklistout
):
2721 def CheckSumSupportAlt(checkfor
, guaranteed
=True):
2724 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2725 except AttributeError:
2726 hash_list
= sorted(list(hashlib
.algorithms
));
2729 hash_list
= sorted(list(hashlib
.algorithms_available
));
2730 except AttributeError:
2731 hash_list
= sorted(list(hashlib
.algorithms
));
2732 checklistout
= hash_list
;
2733 if(checkfor
in checklistout
):
2738 def PackArchiveFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2739 formatspecs
= FormatSpecsListToDict(formatspecs
);
2740 advancedlist
= formatspecs
['use_advanced_list'];
2741 altinode
= formatspecs
['use_alt_inode'];
2742 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2743 outfile
= RemoveWindowsPath(outfile
);
2744 checksumtype
= checksumtype
.lower();
2745 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2746 checksumtype
="crc32";
2747 if(checksumtype
=="none"):
2749 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2750 compression
= "auto";
2751 if(compression
not in compressionlist
and compression
is None):
2752 compression
= "auto";
2754 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2755 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2756 if(os
.path
.exists(outfile
)):
2759 except OSError as e
:
2764 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2766 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2769 fbasename
= os
.path
.splitext(outfile
)[0];
2770 fextname
= os
.path
.splitext(outfile
)[1];
2771 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2772 catver
= formatspecs
['format_ver'];
2773 fileheaderver
= str(int(catver
.replace(".", "")));
2776 for line
in sys
.stdin
:
2777 infilelist
.append(line
.strip());
2778 infilelist
= list(filter(None, infilelist
));
2779 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
2780 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
2782 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
2783 for line
in finfile
:
2784 infilelist
.append(line
.strip());
2785 infilelist
= list(filter(None, infilelist
));
2787 if(isinstance(infiles
, (list, tuple, ))):
2788 infilelist
= list(filter(None, infiles
));
2789 elif(isinstance(infiles
, (str, ))):
2790 infilelist
= list(filter(None, [infiles
]));
2792 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
2794 GetDirList
= ListDir(infilelist
, followlink
, False);
2802 inodetocatinode
= {};
2803 numfiles
= int(len(GetDirList
));
2804 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
2805 for curfname
in GetDirList
:
2806 if(re
.findall(r
"^[.|/]", curfname
)):
2809 fname
= "./"+curfname
;
2811 VerbosePrintOut(fname
);
2812 if(not followlink
or followlink
is None):
2813 fstatinfo
= os
.lstat(fname
);
2815 fstatinfo
= os
.stat(fname
);
2816 fpremode
= fstatinfo
.st_mode
;
2817 finode
= fstatinfo
.st_ino
;
2818 flinkcount
= fstatinfo
.st_nlink
;
2820 if(stat
.S_ISREG(fpremode
)):
2822 elif(stat
.S_ISLNK(fpremode
)):
2824 elif(stat
.S_ISCHR(fpremode
)):
2826 elif(stat
.S_ISBLK(fpremode
)):
2828 elif(stat
.S_ISDIR(fpremode
)):
2830 elif(stat
.S_ISFIFO(fpremode
)):
2832 elif(stat
.S_ISSOCK(fpremode
)):
2834 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
2836 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
2838 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
2843 fcurfid
= format(int(curfid
), 'x').lower();
2844 if(not followlink
and finode
!=0):
2846 if(finode
in inodelist
):
2848 flinkname
= inodetofile
[finode
];
2850 fcurinode
= format(int(finode
), 'x').lower();
2852 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
2853 if(finode
not in inodelist
):
2854 inodelist
.append(finode
);
2855 inodetofile
.update({finode
: fname
});
2856 inodetocatinode
.update({finode
: curinode
});
2858 fcurinode
= format(int(finode
), 'x').lower();
2860 fcurinode
= format(int(curinode
), 'x').lower();
2861 curinode
= curinode
+ 1;
2863 fcurinode
= format(int(curinode
), 'x').lower();
2864 curinode
= curinode
+ 1;
2865 curfid
= curfid
+ 1;
2867 flinkname
= os
.readlink(fname
);
2868 fdev
= fstatinfo
.st_dev
;
2869 getfdev
= GetDevMajorMinor(fdev
);
2870 fdev_minor
= getfdev
[0];
2871 fdev_major
= getfdev
[1];
2872 frdev
= fstatinfo
.st_dev
;
2873 if(hasattr(fstatinfo
, "st_rdev")):
2874 frdev
= fstatinfo
.st_rdev
;
2876 frdev
= fstatinfo
.st_dev
;
2877 getfrdev
= GetDevMajorMinor(frdev
);
2878 frdev_minor
= getfrdev
[0];
2879 frdev_major
= getfrdev
[1];
2880 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2881 fsize
= format(int("0"), 'x').lower();
2882 elif(ftype
==0 or ftype
==7):
2883 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
2885 fsize
= format(int(fstatinfo
.st_size
)).lower();
2886 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
2887 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
2888 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2889 if(hasattr(fstatinfo
, "st_birthtime")):
2890 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
2892 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2893 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
2894 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
2895 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
2896 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
2897 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
2902 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
2903 funame
= userinfo
.pw_name
;
2912 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
2913 fgname
= groupinfo
.gr_name
;
2918 fdev_minor
= format(int(fdev_minor
), 'x').lower();
2919 fdev_major
= format(int(fdev_major
), 'x').lower();
2920 frdev_minor
= format(int(frdev_minor
), 'x').lower();
2921 frdev_major
= format(int(frdev_major
), 'x').lower();
2922 finode
= format(int(finode
), 'x').lower();
2923 flinkcount
= format(int(flinkcount
), 'x').lower();
2924 if(hasattr(fstatinfo
, "st_file_attributes")):
2925 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
2927 fwinattributes
= format(int(0), 'x').lower();
2929 fcsize
= format(int(0), 'x').lower();
2930 fcontents
= BytesIO();
2931 if(ftype
==0 or ftype
==7):
2932 with
open(fname
, "rb") as fpc
:
2933 shutil
.copyfileobj(fpc
, fcontents
);
2934 if(not compresswholefile
):
2935 fcontents
.seek(0, 2);
2936 ucfsize
= fcontents
.tell();
2937 fcontents
.seek(0, 0);
2938 if(compression
=="auto"):
2939 ilsize
= len(compressionlistalt
);
2942 while(ilmin
< ilsize
):
2943 cfcontents
= BytesIO();
2944 shutil
.copyfileobj(fcontents
, cfcontents
);
2945 fcontents
.seek(0, 0);
2946 cfcontents
.seek(0, 0);
2947 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2949 cfcontents
.seek(0, 2);
2950 ilcsize
.append(cfcontents
.tell());
2954 ilcsize
.append(sys
.maxint
);
2955 except AttributeError:
2956 ilcsize
.append(sys
.maxsize
);
2958 ilcmin
= ilcsize
.index(min(ilcsize
));
2959 compression
= compressionlistalt
[ilcmin
];
2960 fcontents
.seek(0, 0);
2961 cfcontents
= BytesIO();
2962 shutil
.copyfileobj(fcontents
, cfcontents
);
2963 cfcontents
.seek(0, 0);
2964 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2965 cfcontents
.seek(0, 2);
2966 cfsize
= cfcontents
.tell();
2967 if(ucfsize
> cfsize
):
2968 fcsize
= format(int(cfsize
), 'x').lower();
2969 fcompression
= compression
;
2971 fcontents
= cfcontents
;
2972 if(fcompression
=="none"):
2974 if(followlink
and (ftype
==1 or ftype
==2)):
2975 flstatinfo
= os
.stat(flinkname
);
2976 with
open(flinkname
, "rb") as fpc
:
2977 shutil
.copyfileobj(fpc
, fcontents
);
2978 if(not compresswholefile
):
2979 fcontents
.seek(0, 2);
2980 ucfsize
= fcontents
.tell();
2981 fcontents
.seek(0, 0);
2982 if(compression
=="auto"):
2983 ilsize
= len(compressionlistalt
);
2986 while(ilmin
< ilsize
):
2987 cfcontents
= BytesIO();
2988 shutil
.copyfileobj(fcontents
, cfcontents
);
2989 fcontents
.seek(0, 0);
2990 cfcontents
.seek(0, 0);
2991 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2993 cfcontents
.seek(0, 2);
2994 ilcsize
.append(cfcontents
.tell());
2998 ilcsize
.append(sys
.maxint
);
2999 except AttributeError:
3000 ilcsize
.append(sys
.maxsize
);
3002 ilcmin
= ilcsize
.index(min(ilcsize
));
3003 compression
= compressionlistalt
[ilcmin
];
3004 fcontents
.seek(0, 0);
3005 cfcontents
= BytesIO();
3006 shutil
.copyfileobj(fcontents
, cfcontents
);
3007 cfcontents
.seek(0, 0);
3008 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3009 cfcontents
.seek(0, 2);
3010 cfsize
= cfcontents
.tell();
3011 if(ucfsize
> cfsize
):
3012 fcsize
= format(int(cfsize
), 'x').lower();
3013 fcompression
= compression
;
3015 fcontents
= cfcontents
;
3016 fcontents
.seek(0, 0);
3017 ftypehex
= format(ftype
, 'x').lower();
3018 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3019 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3022 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3023 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3024 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3027 os
.fsync(catfp
.fileno());
3028 except io
.UnsupportedOperation
:
3030 except AttributeError:
3032 except OSError as e
:
3036 if(hasattr(sys
.stdout
, "buffer")):
3037 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3039 shutil
.copyfileobj(catfp
, sys
.stdout
);
3040 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3041 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3043 upload_file_to_internet_file(catfp
, outfile
);
3051 create_alias_function("Pack", __file_format_name__
, "", PackArchiveFile
);
3053 if(hasattr(shutil
, "register_archive_format")):
3054 def PackArchiveFileFunc(archive_name
, source_dir
, **kwargs
):
3055 return PackArchiveFile(source_dir
, archive_name
, False, "auto", True, None, False, "crc32", [], __file_format_dict__
['format_delimiter'], False, False);
3056 create_alias_function("Pack", __file_format_name__
, "Func", PackArchiveFileFunc
);
3058 def PackArchiveFileFromDirList(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3059 formatspecs
= FormatSpecsListToDict(formatspecs
);
3060 return PackArchiveFile(infiles
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3062 create_alias_function("Pack", __file_format_name__
, "FromDirList", PackArchiveFileFromDirList
);
3064 def PackArchiveFileFromTarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3065 formatspecs
= FormatSpecsListToDict(formatspecs
);
3066 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3067 outfile
= RemoveWindowsPath(outfile
);
3068 checksumtype
= checksumtype
.lower();
3069 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3070 checksumtype
="crc32";
3071 if(checksumtype
=="none"):
3073 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3074 compression
= "auto";
3075 if(compression
not in compressionlist
and compression
is None):
3076 compression
= "auto";
3078 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3079 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3080 if(os
.path
.exists(outfile
)):
3083 except OSError as e
:
3088 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3090 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3093 fbasename
= os
.path
.splitext(outfile
)[0];
3094 fextname
= os
.path
.splitext(outfile
)[1];
3095 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3096 catver
= formatspecs
['format_ver'];
3097 fileheaderver
= str(int(catver
.replace(".", "")));
3103 inodetocatinode
= {};
3106 if(hasattr(sys
.stdin
, "buffer")):
3107 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3109 shutil
.copyfileobj(sys
.stdin
, infile
);
3114 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3115 infile
= download_file_from_internet_file(infile
);
3120 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3122 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
3124 if(not tarfile
.TarFileCheck(infile
)):
3126 except AttributeError:
3127 if(not TarFileCheck(infile
)):
3132 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3133 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
3135 tarfp
= tarfile
.open(infile
, "r");
3136 except FileNotFoundError
:
3138 numfiles
= int(len(tarfp
.getmembers()));
3139 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3140 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
3141 if(re
.findall(r
"^[.|/]", member
.name
)):
3142 fname
= member
.name
;
3144 fname
= "./"+member
.name
;
3146 VerbosePrintOut(fname
);
3147 fpremode
= member
.mode
;
3148 ffullmode
= member
.mode
;
3152 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3154 elif(member
.isdev()):
3155 ffullmode
= member
.mode
;
3157 elif(member
.islnk()):
3158 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3160 elif(member
.issym()):
3161 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
3163 elif(member
.ischr()):
3164 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
3166 elif(member
.isblk()):
3167 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
3169 elif(member
.isdir()):
3170 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
3172 elif(member
.isfifo()):
3173 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
3175 elif(member
.issparse()):
3176 ffullmode
= member
.mode
;
3179 ffullmode
= member
.mode
;
3182 fcurfid
= format(int(curfid
), 'x').lower();
3183 fcurinode
= format(int(curfid
), 'x').lower();
3184 curfid
= curfid
+ 1;
3186 flinkname
= member
.linkname
;
3187 fdev_minor
= format(int(member
.devminor
), 'x').lower();
3188 fdev_major
= format(int(member
.devmajor
), 'x').lower();
3189 frdev_minor
= format(int(member
.devminor
), 'x').lower();
3190 frdev_major
= format(int(member
.devmajor
), 'x').lower();
3191 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
3192 fsize
= format(int("0"), 'x').lower();
3193 elif(ftype
==0 or ftype
==7):
3194 fsize
= format(int(member
.size
), 'x').lower();
3196 fsize
= format(int(member
.size
), 'x').lower();
3197 fatime
= format(int(member
.mtime
), 'x').lower();
3198 fmtime
= format(int(member
.mtime
), 'x').lower();
3199 fctime
= format(int(member
.mtime
), 'x').lower();
3200 fbtime
= format(int(member
.mtime
), 'x').lower();
3201 fmode
= format(int(ffullmode
), 'x').lower();
3202 fchmode
= format(int(stat
.S_IMODE(ffullmode
)), 'x').lower();
3203 ftypemod
= format(int(stat
.S_IFMT(ffullmode
)), 'x').lower();
3204 fuid
= format(int(member
.uid
), 'x').lower();
3205 fgid
= format(int(member
.gid
), 'x').lower();
3206 funame
= member
.uname
;
3207 fgname
= member
.gname
;
3208 flinkcount
= format(int(flinkcount
), 'x').lower();
3209 fwinattributes
= format(int(0), 'x').lower();
3211 fcsize
= format(int(0), 'x').lower();
3212 fcontents
= BytesIO();
3213 if(ftype
==0 or ftype
==7):
3214 with tarfp
.extractfile(member
) as fpc
:
3215 shutil
.copyfileobj(fpc
, fcontents
);
3216 if(not compresswholefile
):
3217 fcontents
.seek(0, 2);
3218 ucfsize
= fcontents
.tell();
3219 fcontents
.seek(0, 0);
3220 if(compression
=="auto"):
3221 ilsize
= len(compressionlistalt
);
3224 while(ilmin
< ilsize
):
3225 cfcontents
= BytesIO();
3226 shutil
.copyfileobj(fcontents
, cfcontents
);
3227 fcontents
.seek(0, 0);
3228 cfcontents
.seek(0, 0);
3229 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3231 cfcontents
.seek(0, 2);
3232 ilcsize
.append(cfcontents
.tell());
3236 ilcsize
.append(sys
.maxint
);
3237 except AttributeError:
3238 ilcsize
.append(sys
.maxsize
);
3240 ilcmin
= ilcsize
.index(min(ilcsize
));
3241 compression
= compressionlistalt
[ilcmin
];
3242 fcontents
.seek(0, 0);
3243 cfcontents
= BytesIO();
3244 shutil
.copyfileobj(fcontents
, cfcontents
);
3245 cfcontents
.seek(0, 0);
3246 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3247 cfcontents
.seek(0, 2);
3248 cfsize
= cfcontents
.tell();
3249 if(ucfsize
> cfsize
):
3250 fcsize
= format(int(cfsize
), 'x').lower();
3251 fcompression
= compression
;
3253 fcontents
= cfcontents
;
3254 if(fcompression
=="none"):
3256 fcontents
.seek(0, 0);
3257 ftypehex
= format(ftype
, 'x').lower();
3258 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3259 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3262 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3263 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3264 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3267 os
.fsync(catfp
.fileno());
3268 except io
.UnsupportedOperation
:
3270 except AttributeError:
3272 except OSError as e
:
3276 if(hasattr(sys
.stdout
, "buffer")):
3277 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3279 shutil
.copyfileobj(catfp
, sys
.stdout
);
3280 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3281 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3283 upload_file_to_internet_file(catfp
, outfile
);
3291 create_alias_function("Pack", __file_format_name__
, "FromTarFile", PackArchiveFileFromTarFile
);
3293 def PackArchiveFileFromZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3294 formatspecs
= FormatSpecsListToDict(formatspecs
);
3295 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3296 outfile
= RemoveWindowsPath(outfile
);
3297 checksumtype
= checksumtype
.lower();
3298 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3299 checksumtype
="crc32";
3300 if(checksumtype
=="none"):
3302 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3303 compression
= "auto";
3304 if(compression
not in compressionlist
and compression
is None):
3305 compression
= "auto";
3307 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3308 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3309 if(os
.path
.exists(outfile
)):
3312 except OSError as e
:
3317 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3319 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3322 fbasename
= os
.path
.splitext(outfile
)[0];
3323 fextname
= os
.path
.splitext(outfile
)[1];
3324 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3325 catver
= formatspecs
['format_ver'];
3326 fileheaderver
= str(int(catver
.replace(".", "")));
3332 inodetocatinode
= {};
3335 if(hasattr(sys
.stdin
, "buffer")):
3336 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3338 shutil
.copyfileobj(sys
.stdin
, infile
);
3343 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3344 infile
= download_file_from_internet_file(infile
);
3349 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3353 if(not zipfile
.is_zipfile(infile
)):
3356 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
3357 except FileNotFoundError
:
3359 ziptest
= zipfp
.testzip();
3361 VerbosePrintOut("Bad file found!");
3362 numfiles
= int(len(zipfp
.infolist()));
3363 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3364 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
3365 if(re
.findall(r
"^[.|/]", member
.filename
)):
3366 fname
= member
.filename
;
3368 fname
= "./"+member
.filename
;
3369 zipinfo
= zipfp
.getinfo(member
.filename
);
3371 VerbosePrintOut(fname
);
3372 if(not member
.is_dir()):
3373 fpremode
= int(stat
.S_IFREG
+ 438);
3374 elif(member
.is_dir()):
3375 fpremode
= int(stat
.S_IFDIR
+ 511);
3378 if(not member
.is_dir()):
3380 elif(member
.is_dir()):
3383 fcurfid
= format(int(curfid
), 'x').lower();
3384 fcurinode
= format(int(curfid
), 'x').lower();
3385 curfid
= curfid
+ 1;
3386 fdev_minor
= format(int(0), 'x').lower();
3387 fdev_major
= format(int(0), 'x').lower();
3388 frdev_minor
= format(int(0), 'x').lower();
3389 frdev_major
= format(int(0), 'x').lower();
3391 fsize
= format(int("0"), 'x').lower();
3393 fsize
= format(int(member
.file_size
), 'x').lower();
3395 fsize
= format(int(member
.file_size
), 'x').lower();
3396 fatime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3397 fmtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3398 fctime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3399 fbtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3400 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
3401 fwinattributes
= format(int(zipinfo
.external_attr
), 'x').lower();
3402 if(not member
.is_dir()):
3403 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3404 fchmode
= stat
.S_IMODE(fmode
);
3405 ftypemod
= stat
.S_IFMT(fmode
);
3406 elif(member
.is_dir()):
3407 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3408 fchmode
= stat
.S_IMODE(fmode
);
3409 ftypemod
= stat
.S_IFMT(fmode
);
3410 elif(zipinfo
.create_system
==3):
3411 fwinattributes
= format(int(0), 'x').lower();
3412 fmode
= format(int(zipinfo
.external_attr
), 'x').lower();
3413 fchmode
= stat
.S_IMODE(fmode
);
3414 ftypemod
= stat
.S_IFMT(fmode
);
3416 fwinattributes
= format(int(0), 'x').lower();
3417 if(not member
.is_dir()):
3418 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3419 fchmode
= stat
.S_IMODE(fmode
);
3420 ftypemod
= stat
.S_IFMT(fmode
);
3421 elif(member
.is_dir()):
3422 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3423 fchmode
= stat
.S_IMODE(fmode
);
3424 ftypemod
= stat
.S_IFMT(fmode
);
3426 fcsize
= format(int(0), 'x').lower();
3428 fuid
= format(int(os
.getuid()), 'x').lower();
3429 except AttributeError:
3430 fuid
= format(int(0), 'x').lower();
3432 fuid
= format(int(0), 'x').lower();
3434 fgid
= format(int(os
.getgid()), 'x').lower();
3435 except AttributeError:
3436 fgid
= format(int(0), 'x').lower();
3438 fgid
= format(int(0), 'x').lower();
3442 userinfo
= pwd
.getpwuid(os
.getuid());
3443 funame
= userinfo
.pw_name
;
3446 except AttributeError:
3454 groupinfo
= grp
.getgrgid(os
.getgid());
3455 fgname
= groupinfo
.gr_name
;
3458 except AttributeError:
3462 fcontents
= BytesIO();
3464 fcontents
.write(zipfp
.read(member
.filename
));
3465 if(not compresswholefile
):
3466 fcontents
.seek(0, 2);
3467 ucfsize
= fcontents
.tell();
3468 fcontents
.seek(0, 0);
3469 if(compression
=="auto"):
3470 ilsize
= len(compressionlistalt
);
3473 while(ilmin
< ilsize
):
3474 cfcontents
= BytesIO();
3475 shutil
.copyfileobj(fcontents
, cfcontents
);
3476 fcontents
.seek(0, 0);
3477 cfcontents
.seek(0, 0);
3478 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3479 cfcontents
.seek(0, 2);
3480 ilcsize
.append(cfcontents
.tell());
3483 ilcmin
= ilcsize
.index(min(ilcsize
));
3484 compression
= compressionlistalt
[ilcmin
];
3485 fcontents
.seek(0, 0);
3486 cfcontents
= BytesIO();
3487 shutil
.copyfileobj(fcontents
, cfcontents
);
3488 cfcontents
.seek(0, 0);
3489 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3490 cfcontents
.seek(0, 2);
3491 cfsize
= cfcontents
.tell();
3492 if(ucfsize
> cfsize
):
3493 fcsize
= format(int(cfsize
), 'x').lower();
3494 fcompression
= compression
;
3496 fcontents
= cfcontents
;
3497 if(fcompression
=="none"):
3499 fcontents
.seek(0, 0);
3500 ftypehex
= format(ftype
, 'x').lower();
3501 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3502 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3505 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3506 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3507 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3510 os
.fsync(catfp
.fileno());
3511 except io
.UnsupportedOperation
:
3513 except AttributeError:
3515 except OSError as e
:
3519 if(hasattr(sys
.stdout
, "buffer")):
3520 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3522 shutil
.copyfileobj(catfp
, sys
.stdout
);
3523 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3524 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3526 upload_file_to_internet_file(catfp
, outfile
);
3534 create_alias_function("Pack", __file_format_name__
, "FromZipFile", PackArchiveFileFromZipFile
);
3536 if(not rarfile_support
):
3537 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3540 if(rarfile_support
):
3541 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3542 formatspecs
= FormatSpecsListToDict(formatspecs
);
3543 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3544 outfile
= RemoveWindowsPath(outfile
);
3545 checksumtype
= checksumtype
.lower();
3546 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3547 checksumtype
="crc32";
3548 if(checksumtype
=="none"):
3550 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3551 compression
= "auto";
3552 if(compression
not in compressionlist
and compression
is None):
3553 compression
= "auto";
3555 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3556 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3557 if(os
.path
.exists(outfile
)):
3560 except OSError as e
:
3565 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3567 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3570 fbasename
= os
.path
.splitext(outfile
)[0];
3571 fextname
= os
.path
.splitext(outfile
)[1];
3572 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3573 catver
= formatspecs
['format_ver'];
3574 fileheaderver
= str(int(catver
.replace(".", "")));
3580 inodetocatinode
= {};
3581 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3583 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
3585 rarfp
= rarfile
.RarFile(infile
, "r");
3586 rartest
= rarfp
.testrar();
3588 VerbosePrintOut("Bad file found!");
3589 numfiles
= int(len(rarfp
.infolist()));
3590 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3593 os
.fsync(catfp
.fileno());
3594 except io
.UnsupportedOperation
:
3596 except AttributeError:
3598 except OSError as e
:
3600 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
3603 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
3606 member
.external_attr
3608 except AttributeError:
3610 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
3613 member
.external_attr
3615 except AttributeError:
3620 if(re
.findall(r
"^[.|/]", member
.filename
)):
3621 fname
= member
.filename
;
3623 fname
= "./"+member
.filename
;
3624 rarinfo
= rarfp
.getinfo(member
.filename
);
3626 VerbosePrintOut(fname
);
3627 if(is_unix
and member
.external_attr
!=0):
3628 fpremode
= int(member
.external_attr
);
3629 elif(member
.is_file()):
3630 fpremode
= int(stat
.S_IFREG
+ 438);
3631 elif(member
.is_symlink()):
3632 fpremode
= int(stat
.S_IFLNK
+ 438);
3633 elif(member
.is_dir()):
3634 fpremode
= int(stat
.S_IFDIR
+ 511);
3635 if(is_windows
and member
.external_attr
!=0):
3636 fwinattributes
= format(int(member
.external_attr
), 'x').lower();
3638 fwinattributes
= format(int(0), 'x').lower();
3640 fcsize
= format(int(0), 'x').lower();
3643 if(member
.is_file()):
3645 elif(member
.is_symlink()):
3647 elif(member
.is_dir()):
3651 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
3652 fcurfid
= format(int(curfid
), 'x').lower();
3653 fcurinode
= format(int(curfid
), 'x').lower();
3654 curfid
= curfid
+ 1;
3655 fdev_minor
= format(int(0), 'x').lower();
3656 fdev_major
= format(int(0), 'x').lower();
3657 frdev_minor
= format(int(0), 'x').lower();
3658 frdev_major
= format(int(0), 'x').lower();
3660 fsize
= format(int("0"), 'x').lower();
3662 fsize
= format(int(member
.file_size
), 'x').lower();
3664 fsize
= format(int(member
.file_size
), 'x').lower();
3667 fatime
= format(int(member
.atime
.timestamp()), 'x').lower();
3669 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3670 except AttributeError:
3671 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3672 fmtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3675 fctime
= format(int(member
.ctime
.timestamp()), 'x').lower();
3677 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3678 except AttributeError:
3679 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3680 fbtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3681 if(is_unix
and member
.external_attr
!=0):
3682 fmode
= format(int(member
.external_attr
), 'x').lower();
3683 fchmode
= format(int(stat
.S_IMODE(member
.external_attr
)), 'x').lower();
3684 ftypemod
= format(int(stat
.S_IFMT(member
.external_attr
)), 'x').lower();
3685 elif(member
.is_file()):
3686 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3687 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3688 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3689 elif(member
.is_symlink()):
3690 fmode
= format(int(stat
.S_IFLNK
+ 438), 'x').lower();
3691 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3692 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3693 elif(member
.is_dir()):
3694 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3695 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3696 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3698 fuid
= format(int(os
.getuid()), 'x').lower();
3699 except AttributeError:
3700 fuid
= format(int(0), 'x').lower();
3702 fuid
= format(int(0), 'x').lower();
3704 fgid
= format(int(os
.getgid()), 'x').lower();
3705 except AttributeError:
3706 fgid
= format(int(0), 'x').lower();
3708 fgid
= format(int(0), 'x').lower();
3712 userinfo
= pwd
.getpwuid(os
.getuid());
3713 funame
= userinfo
.pw_name
;
3716 except AttributeError:
3724 groupinfo
= grp
.getgrgid(os
.getgid());
3725 fgname
= groupinfo
.gr_name
;
3728 except AttributeError:
3732 fcontents
= BytesIO();
3734 fcontents
.write(rarfp
.read(member
.filename
));
3735 if(not compresswholefile
):
3736 fcontents
.seek(0, 2);
3737 ucfsize
= fcontents
.tell();
3738 fcontents
.seek(0, 0);
3739 if(compression
=="auto"):
3740 ilsize
= len(compressionlistalt
);
3743 while(ilmin
< ilsize
):
3744 cfcontents
= BytesIO();
3745 shutil
.copyfileobj(fcontents
, cfcontents
);
3746 fcontents
.seek(0, 0);
3747 cfcontents
.seek(0, 0);
3748 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3750 cfcontents
.seek(0, 2);
3751 ilcsize
.append(cfcontents
.tell());
3755 ilcsize
.append(sys
.maxint
);
3756 except AttributeError:
3757 ilcsize
.append(sys
.maxsize
);
3759 ilcmin
= ilcsize
.index(min(ilcsize
));
3760 compression
= compressionlistalt
[ilcmin
];
3761 fcontents
.seek(0, 0);
3762 cfcontents
= BytesIO();
3763 shutil
.copyfileobj(fcontents
, cfcontents
);
3764 cfcontents
.seek(0, 0);
3765 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3766 cfcontents
.seek(0, 2);
3767 cfsize
= cfcontents
.tell();
3768 if(ucfsize
> cfsize
):
3769 fcsize
= format(int(cfsize
), 'x').lower();
3770 fcompression
= compression
;
3772 fcontents
= cfcontents
;
3773 if(fcompression
=="none"):
3775 fcontents
.seek(0, 0);
3776 ftypehex
= format(ftype
, 'x').lower();
3777 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3778 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3781 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3782 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3783 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3786 os
.fsync(catfp
.fileno());
3787 except io
.UnsupportedOperation
:
3789 except AttributeError:
3791 except OSError as e
:
3795 if(hasattr(sys
.stdout
, "buffer")):
3796 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3798 shutil
.copyfileobj(catfp
, sys
.stdout
);
3799 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3800 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3802 upload_file_to_internet_file(catfp
, outfile
);
3810 create_alias_function("Pack", __file_format_name__
, "FromRarFile", PackArchiveFileFromRarFile
);
3812 if(not py7zr_support
):
3813 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3817 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3818 formatspecs
= FormatSpecsListToDict(formatspecs
);
3819 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3820 outfile
= RemoveWindowsPath(outfile
);
3821 checksumtype
= checksumtype
.lower();
3822 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3823 checksumtype
="crc32";
3824 if(checksumtype
=="none"):
3826 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3827 compression
= "auto";
3828 if(compression
not in compressionlist
and compression
is None):
3829 compression
= "auto";
3831 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3832 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3833 if(os
.path
.exists(outfile
)):
3836 except OSError as e
:
3841 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3843 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3846 fbasename
= os
.path
.splitext(outfile
)[0];
3847 fextname
= os
.path
.splitext(outfile
)[1];
3848 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3849 catver
= formatspecs
['format_ver'];
3850 fileheaderver
= str(int(catver
.replace(".", "")));
3856 inodetocatinode
= {};
3857 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3859 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
3860 file_content
= szpfp
.readall();
3861 #sztest = szpfp.testzip();
3862 sztestalt
= szpfp
.test();
3864 VerbosePrintOut("Bad file found!");
3865 numfiles
= int(len(szpfp
.list()));
3866 AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3867 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
3868 if(re
.findall(r
"^[.|/]", member
.filename
)):
3869 fname
= member
.filename
;
3871 fname
= "./"+member
.filename
;
3873 VerbosePrintOut(fname
);
3874 if(not member
.is_directory
):
3875 fpremode
= int(stat
.S_IFREG
+ 438);
3876 elif(member
.is_directory
):
3877 fpremode
= int(stat
.S_IFDIR
+ 511);
3878 fwinattributes
= format(int(0), 'x').lower();
3880 fcsize
= format(int(0), 'x').lower();
3883 if(member
.is_directory
):
3888 fcurfid
= format(int(curfid
), 'x').lower();
3889 fcurinode
= format(int(curfid
), 'x').lower();
3890 curfid
= curfid
+ 1;
3891 fdev_minor
= format(int(0), 'x').lower();
3892 fdev_major
= format(int(0), 'x').lower();
3893 frdev_minor
= format(int(0), 'x').lower();
3894 frdev_major
= format(int(0), 'x').lower();
3896 fsize
= format(int("0"), 'x').lower();
3897 fatime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3898 fmtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3899 fctime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3900 fbtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3901 if(member
.is_directory
):
3902 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3903 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3904 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3906 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3907 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3908 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3910 fuid
= format(int(os
.getuid()), 'x').lower();
3911 except AttributeError:
3912 fuid
= format(int(0), 'x').lower();
3914 fuid
= format(int(0), 'x').lower();
3916 fgid
= format(int(os
.getgid()), 'x').lower();
3917 except AttributeError:
3918 fgid
= format(int(0), 'x').lower();
3920 fgid
= format(int(0), 'x').lower();
3924 userinfo
= pwd
.getpwuid(os
.getuid());
3925 funame
= userinfo
.pw_name
;
3928 except AttributeError:
3936 groupinfo
= grp
.getgrgid(os
.getgid());
3937 fgname
= groupinfo
.gr_name
;
3940 except AttributeError:
3944 fcontents
= BytesIO();
3946 fcontents
.write(file_content
[member
.filename
].read());
3947 fsize
= format(fcontents
.tell(), 'x').lower();
3948 file_content
[member
.filename
].close();
3949 if(not compresswholefile
):
3950 fcontents
.seek(0, 2);
3951 ucfsize
= fcontents
.tell();
3952 fcontents
.seek(0, 0);
3953 if(compression
=="auto"):
3954 ilsize
= len(compressionlistalt
);
3957 while(ilmin
< ilsize
):
3958 cfcontents
= BytesIO();
3959 shutil
.copyfileobj(fcontents
, cfcontents
);
3960 fcontents
.seek(0, 0);
3961 cfcontents
.seek(0, 0);
3962 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3964 cfcontents
.seek(0, 2);
3965 ilcsize
.append(cfcontents
.tell());
3969 ilcsize
.append(sys
.maxint
);
3970 except AttributeError:
3971 ilcsize
.append(sys
.maxsize
);
3973 ilcmin
= ilcsize
.index(min(ilcsize
));
3974 compression
= compressionlistalt
[ilcmin
];
3975 fcontents
.seek(0, 0);
3976 cfcontents
= BytesIO();
3977 shutil
.copyfileobj(fcontents
, cfcontents
);
3978 cfcontents
.seek(0, 0);
3979 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3980 cfcontents
.seek(0, 2);
3981 cfsize
= cfcontents
.tell();
3982 if(ucfsize
> cfsize
):
3983 fcsize
= format(int(cfsize
), 'x').lower();
3984 fcompression
= compression
;
3986 fcontents
= cfcontents
;
3987 if(fcompression
=="none"):
3989 fcontents
.seek(0, 0);
3990 ftypehex
= format(ftype
, 'x').lower();
3991 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3992 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3995 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3996 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3997 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
4000 os
.fsync(catfp
.fileno());
4001 except io
.UnsupportedOperation
:
4003 except AttributeError:
4005 except OSError as e
:
4009 if(hasattr(sys
.stdout
, "buffer")):
4010 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
4012 shutil
.copyfileobj(catfp
, sys
.stdout
);
4013 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
4014 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
4016 upload_file_to_internet_file(catfp
, outfile
);
4024 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFile", PackArchiveFileFromSevenZipFile
);
4026 def PackArchiveFileFromInFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4027 formatspecs
= FormatSpecsListToDict(formatspecs
);
4028 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4030 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4031 if(checkcompressfile
=="tarfile"):
4032 return PackArchiveFileFromTarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4033 elif(checkcompressfile
=="zipfile"):
4034 return PackArchiveFileFromZipFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4035 elif(checkcompressfile
=="catfile"):
4036 return RePackArchiveFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, False, 0, 0, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
4037 elif(rarfile_support
and checkcompressfile
=="rarfile"):
4038 return PackArchiveFileFromRarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4039 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
4040 return PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4045 create_alias_function("Pack", __file_format_name__
, "FromInFile", PackArchiveFileFromInFile
);
4047 def ArchiveFileSeekToFileNum(infile
, seekto
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4048 formatspecs
= FormatSpecsListToDict(formatspecs
);
4049 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4052 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4053 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4054 if(checkcompressfile
=="tarfile"):
4055 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4056 if(checkcompressfile
=="zipfile"):
4057 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4058 if(rarfile_support
and checkcompressfile
=="rarfile"):
4059 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4060 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4061 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4062 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4069 if(hasattr(sys
.stdin
, "buffer")):
4070 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4072 shutil
.copyfileobj(sys
.stdin
, catfp
);
4074 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4078 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4079 catfp
= download_file_from_internet_file(infile
);
4081 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4086 infile
= RemoveWindowsPath(infile
);
4087 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4088 if(checkcompressfile
=="tarfile"):
4089 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4090 if(checkcompressfile
=="zipfile"):
4091 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4092 if(rarfile_support
and checkcompressfile
=="rarfile"):
4093 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4094 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4095 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4096 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4098 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4099 if(not compresscheck
):
4100 fextname
= os
.path
.splitext(infile
)[1];
4101 if(fextname
==".gz"):
4102 compresscheck
= "gzip";
4103 elif(fextname
==".bz2"):
4104 compresscheck
= "bzip2";
4105 elif(fextname
==".zst"):
4106 compresscheck
= "zstd";
4107 elif(fextname
==".lz4" or fextname
==".clz4"):
4108 compresscheck
= "lz4";
4109 elif(fextname
==".lzo" or fextname
==".lzop"):
4110 compresscheck
= "lzo";
4111 elif(fextname
==".lzma" or fextname
==".xz"):
4112 compresscheck
= "lzma";
4115 if(not compresscheck
):
4117 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4122 SeekToEndOfFile(catfp);
4124 SeekToEndOfFile(catfp);
4125 CatSize = catfp.tell();
4126 CatSizeEnd = CatSize;
4134 curloc
= catfp
.tell();
4137 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4139 catfp
.seek(curloc
, 0);
4140 catstring
= catheader
[0];
4141 catversion
= re
.findall(r
"([\d]+)", catstring
);
4142 fprenumfiles
= catheader
[1];
4143 fnumfiles
= int(fprenumfiles
, 16);
4144 fprechecksumtype
= catheader
[2];
4145 fprechecksum
= catheader
[3];
4146 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4147 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4148 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4149 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4150 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4151 fheadtell
= len(fileheader
);
4152 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4153 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4154 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4156 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4157 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4158 if(seekto
>=fnumfiles
):
4159 seekto
= fnumfiles
- 1;
4165 prefhstart
= catfp
.tell();
4166 if(formatspecs
['new_style']):
4167 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4169 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4170 if(len(preheaderdata
)==0):
4172 prefheadsize
= int(preheaderdata
[0], 16);
4173 prefnumfields
= int(preheaderdata
[1], 16);
4174 preftype
= int(preheaderdata
[2], 16);
4175 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4176 prefname
= preheaderdata
[3];
4178 prefname
= "./"+preheaderdata
[3];
4179 prefbasedir
= os
.path
.dirname(prefname
);
4180 preflinkname
= preheaderdata
[4];
4181 prefsize
= int(preheaderdata
[5], 16);
4182 prefatime
= int(preheaderdata
[6], 16);
4183 prefmtime
= int(preheaderdata
[7], 16);
4184 prefctime
= int(preheaderdata
[8], 16);
4185 prefbtime
= int(preheaderdata
[9], 16);
4186 prefmode
= int(preheaderdata
[10], 16);
4187 prefchmode
= stat
.S_IMODE(prefmode
);
4188 preftypemod
= stat
.S_IFMT(prefmode
);
4189 prefwinattributes
= int(preheaderdata
[11], 16);
4190 prefcompression
= preheaderdata
[12];
4191 prefcsize
= int(preheaderdata
[13], 16);
4192 prefuid
= int(preheaderdata
[14], 16);
4193 prefuname
= preheaderdata
[15];
4194 prefgid
= int(preheaderdata
[16], 16);
4195 prefgname
= preheaderdata
[17];
4196 fid
= int(preheaderdata
[18], 16);
4197 finode
= int(preheaderdata
[19], 16);
4198 flinkcount
= int(preheaderdata
[20], 16);
4199 prefdev_minor
= int(preheaderdata
[21], 16);
4200 prefdev_major
= int(preheaderdata
[22], 16);
4201 prefrdev_minor
= int(preheaderdata
[23], 16);
4202 prefrdev_major
= int(preheaderdata
[24], 16);
4203 prefseeknextfile
= preheaderdata
[25];
4204 prefextrasize
= int(preheaderdata
[26], 16);
4205 prefextrafields
= int(preheaderdata
[27], 16);
4206 extrafieldslist
= [];
4208 extraend
= extrastart
+ prefextrafields
;
4209 extrafieldslist
= [];
4210 if(extrastart
<extraend
):
4211 extrafieldslist
.append(preheaderdata
[extrastart
]);
4212 extrastart
= extrastart
+ 1;
4213 prefcs
= preheaderdata
[extrastart
+ 2].lower();
4214 prefccs
= preheaderdata
[extrastart
+ 3].lower();
4215 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
4216 if(prefcs
!=prenewfcs
and not skipchecksum
):
4217 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4218 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4220 valid_archive
= False;
4221 invalid_archive
= True;
4222 prefhend
= catfp
.tell() - 1;
4223 prefcontentstart
= catfp
.tell();
4225 pyhascontents
= False;
4227 if(prefcompression
):
4228 prefcontents
= catfp
.read(prefsize
);
4230 prefcontents
= catfp
.read(prefcsize
);
4231 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4232 pyhascontents
= True;
4233 if(prefccs
!=prenewfccs
and not skipchecksum
):
4234 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4235 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4237 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
4238 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
4239 if(abs(fseeknextasnum
)==0):
4241 catfp
.seek(fseeknextasnum
, 1);
4242 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
4243 fseeknextasnum
= int(prefseeknextfile
);
4244 if(abs(fseeknextasnum
)==0):
4246 catfp
.seek(fseeknextasnum
, 1);
4247 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
4248 fseeknextasnum
= int(prefseeknextfile
);
4249 if(abs(fseeknextasnum
)==0):
4251 catfp
.seek(fseeknextasnum
, 0);
4255 catfp
.seek(seekstart
, 0);
4257 catfheadsize
= int(preheaderdata
[0], 16);
4258 catfnumfields
= int(preheaderdata
[1], 16);
4259 catftype
= int(preheaderdata
[2], 16);
4260 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4261 catfname
= preheaderdata
[3];
4263 catfname
= "./"+preheaderdata
[3];
4264 catflinkname
= preheaderdata
[4];
4265 catfsize
= int(preheaderdata
[5], 16);
4266 catfbasedir
= os
.path
.dirname(catfname
);
4267 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4269 catlist
.update({'catfp': catfp
});
4274 create_alias_function("", __file_format_name__
, "SeekToFileNum", ArchiveFileSeekToFileNum
);
4276 def ArchiveFileSeekToFileName(infile
, seekfile
=None, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4277 formatspecs
= FormatSpecsListToDict(formatspecs
);
4278 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4281 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4282 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4283 if(checkcompressfile
=="tarfile"):
4284 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4285 if(checkcompressfile
=="zipfile"):
4286 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4287 if(rarfile_support
and checkcompressfile
=="rarfile"):
4288 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4289 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4290 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4291 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4298 if(hasattr(sys
.stdin
, "buffer")):
4299 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4301 shutil
.copyfileobj(sys
.stdin
, catfp
);
4303 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4307 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4308 catfp
= download_file_from_internet_file(infile
);
4309 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4315 infile
= RemoveWindowsPath(infile
);
4316 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4317 if(checkcompressfile
=="tarfile"):
4318 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4319 if(checkcompressfile
=="zipfile"):
4320 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4321 if(rarfile_support
and checkcompressfile
=="rarfile"):
4322 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4323 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4324 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4325 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4327 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4328 if(not compresscheck
):
4329 fextname
= os
.path
.splitext(infile
)[1];
4330 if(fextname
==".gz"):
4331 compresscheck
= "gzip";
4332 elif(fextname
==".bz2"):
4333 compresscheck
= "bzip2";
4334 elif(fextname
==".zst"):
4335 compresscheck
= "zstd";
4336 elif(fextname
==".lz4" or fextname
==".clz4"):
4337 compresscheck
= "lz4";
4338 elif(fextname
==".lzo" or fextname
==".lzop"):
4339 compresscheck
= "lzo";
4340 elif(fextname
==".lzma" or fextname
==".xz"):
4341 compresscheck
= "lzma";
4344 if(not compresscheck
):
4346 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4351 SeekToEndOfFile(catfp);
4353 SeekToEndOfFile(catfp);
4354 CatSize = catfp.tell();
4355 CatSizeEnd = CatSize;
4363 curloc
= catfp
.tell();
4366 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4368 catfp
.seek(curloc
, 0);
4369 catstring
= catheader
[0];
4370 catversion
= re
.findall(r
"([\d]+)", catstring
);
4371 fprenumfiles
= catheader
[1];
4372 fnumfiles
= int(fprenumfiles
, 16);
4373 fprechecksumtype
= catheader
[2];
4374 fprechecksum
= catheader
[3];
4375 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4376 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4377 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4378 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4379 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4380 fheadtell
= len(fileheader
);
4381 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4382 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4383 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4385 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4386 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4387 seekto
= fnumfiles
- 1
4392 prefhstart
= catfp
.tell();
4393 if(formatspecs
['new_style']):
4394 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4396 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4397 if(len(preheaderdata
)==0):
4399 prefheadsize
= int(preheaderdata
[0], 16);
4400 prefnumfields
= int(preheaderdata
[1], 16);
4401 preftype
= int(preheaderdata
[2], 16);
4402 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4403 prefname
= preheaderdata
[3];
4405 prefname
= "./"+preheaderdata
[3];
4406 prefbasedir
= os
.path
.dirname(prefname
);
4407 preflinkname
= preheaderdata
[4];
4408 prefsize
= int(preheaderdata
[5], 16);
4409 prefatime
= int(preheaderdata
[6], 16);
4410 prefmtime
= int(preheaderdata
[7], 16);
4411 prefctime
= int(preheaderdata
[8], 16);
4412 prefbtime
= int(preheaderdata
[9], 16);
4413 prefmode
= int(preheaderdata
[10], 16);
4414 prefchmode
= stat
.S_IMODE(prefmode
);
4415 preftypemod
= stat
.S_IFMT(prefmode
);
4416 prefwinattributes
= int(preheaderdata
[11], 16);
4417 prefcompression
= preheaderdata
[12];
4418 prefcsize
= int(preheaderdata
[13], 16);
4419 prefuid
= int(preheaderdata
[14], 16);
4420 prefuname
= preheaderdata
[15];
4421 prefgid
= int(preheaderdata
[16], 16);
4422 prefgname
= preheaderdata
[17];
4423 fid
= int(preheaderdata
[18], 16);
4424 finode
= int(preheaderdata
[19], 16);
4425 flinkcount
= int(preheaderdata
[20], 16);
4426 prefdev_minor
= int(preheaderdata
[21], 16);
4427 prefdev_major
= int(preheaderdata
[22], 16);
4428 prefrdev_minor
= int(preheaderdata
[23], 16);
4429 prefrdev_major
= int(preheaderdata
[24], 16);
4430 prefseeknextfile
= preheaderdata
[25];
4431 prefextrasize
= int(preheaderdata
[26], 16);
4432 prefextrafields
= int(preheaderdata
[27], 16);
4433 extrafieldslist
= [];
4435 extraend
= extrastart
+ prefextrafields
;
4436 extrafieldslist
= [];
4437 if(extrastart
<extraend
):
4438 extrafieldslist
.append(preheaderdata
[extrastart
]);
4439 extrastart
= extrastart
+ 1;
4440 prefcs
= preheaderdata
[extrastart
+ 2].lower();
4441 prefccs
= preheaderdata
[extrastart
+ 3].lower();
4442 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
4443 if(prefcs
!=prenewfcs
and not skipchecksum
):
4444 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4445 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4447 valid_archive
= False;
4448 invalid_archive
= True;
4449 prefhend
= catfp
.tell() - 1;
4450 prefcontentstart
= catfp
.tell();
4452 pyhascontents
= False;
4454 if(prefcompression
):
4455 prefcontents
= catfp
.read(prefsize
);
4457 prefcontents
= catfp
.read(prefcsize
);
4458 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4459 pyhascontents
= True;
4460 if(prefccs
!=prenewfccs
and not skipchecksum
):
4461 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4462 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4464 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
4465 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
4466 if(abs(fseeknextasnum
)==0):
4468 catfp
.seek(fseeknextasnum
, 1);
4469 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
4470 fseeknextasnum
= int(prefseeknextfile
);
4471 if(abs(fseeknextasnum
)==0):
4473 catfp
.seek(fseeknextasnum
, 1);
4474 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
4475 fseeknextasnum
= int(prefseeknextfile
);
4476 if(abs(fseeknextasnum
)==0):
4478 catfp
.seek(fseeknextasnum
, 0);
4483 prefname
= preheaderdata
[2];
4484 if(re
.findall(r
"^[.|/]", preheaderdata
[2])):
4485 prefname
= preheaderdata
[2];
4487 prefname
= "./"+preheaderdata
[2];
4488 if(prefname
==seekfile
):
4491 catfp
.seek(seekstart
, 0);
4493 catfheadsize
= int(preheaderdata
[0], 16);
4494 catfnumfields
= int(preheaderdata
[1], 16);
4495 catftype
= int(preheaderdata
[2], 16);
4496 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4497 catfname
= preheaderdata
[3];
4499 catfname
= "./"+preheaderdata
[3];
4500 catflinkname
= preheaderdata
[4];
4501 catfsize
= int(preheaderdata
[5], 16);
4502 catfbasedir
= os
.path
.dirname(catfname
);
4504 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4508 catlist
.update({'catfp': catfp
});
4513 create_alias_function("", __file_format_name__
, "SeekToFileName", ArchiveFileSeekToFileName
);
4515 def ArchiveFileValidate(infile
, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4516 formatspecs
= FormatSpecsListToDict(formatspecs
);
4518 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4519 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4522 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4523 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4524 if(checkcompressfile
=="tarfile"):
4525 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4526 if(checkcompressfile
=="zipfile"):
4527 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4528 if(rarfile_support
and checkcompressfile
=="rarfile"):
4529 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4530 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4531 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4532 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4539 if(hasattr(sys
.stdin
, "buffer")):
4540 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4542 shutil
.copyfileobj(sys
.stdin
, catfp
);
4544 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4548 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4549 catfp
= download_file_from_internet_file(infile
);
4550 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4556 infile
= RemoveWindowsPath(infile
);
4557 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4558 if(checkcompressfile
=="tarfile"):
4559 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4560 if(checkcompressfile
=="zipfile"):
4561 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4562 if(rarfile_support
and checkcompressfile
=="rarfile"):
4563 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4564 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4565 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4566 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4568 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4569 if(not compresscheck
):
4570 fextname
= os
.path
.splitext(infile
)[1];
4571 if(fextname
==".gz"):
4572 compresscheck
= "gzip";
4573 elif(fextname
==".bz2"):
4574 compresscheck
= "bzip2";
4575 elif(fextname
==".zst"):
4576 compresscheck
= "zstd";
4577 elif(fextname
==".lz4" or fextname
==".clz4"):
4578 compresscheck
= "lz4";
4579 elif(fextname
==".lzo" or fextname
==".lzop"):
4580 compresscheck
= "lzo";
4581 elif(fextname
==".lzma" or fextname
==".xz"):
4582 compresscheck
= "lzma";
4585 if(not compresscheck
):
4587 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4592 SeekToEndOfFile(catfp);
4594 SeekToEndOfFile(catfp);
4595 CatSize = catfp.tell();
4596 CatSizeEnd = CatSize;
4604 curloc
= catfp
.tell();
4607 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4609 catfp
.seek(curloc
, 0);
4610 catstring
= catheader
[0];
4611 catversion
= re
.findall(r
"([\d]+)", catstring
);
4612 fprenumfiles
= catheader
[1];
4613 fnumfiles
= int(fprenumfiles
, 16);
4614 fprechecksumtype
= catheader
[2];
4615 fprechecksum
= catheader
[3];
4617 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4618 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4619 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4620 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4621 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4622 valid_archive
= True;
4623 invalid_archive
= False;
4625 VerbosePrintOut(infile
);
4626 VerbosePrintOut("Number of Records " + str(fnumfiles
));
4627 if(fprechecksum
==catfileheadercshex
):
4629 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4630 VerbosePrintOut("'" + str(fprechecksum
) + "' == " + "'" + str(catfileheadercshex
) + "'");
4633 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4634 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4635 valid_archive
= False;
4636 invalid_archive
= True;
4638 VerbosePrintOut("");
4639 while(il
<fnumfiles
):
4640 catfhstart
= catfp
.tell();
4641 if(formatspecs
['new_style']):
4642 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4644 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4645 if(len(catheaderdata
)==0):
4647 catfheadsize
= int(catheaderdata
[0], 16);
4648 catfnumfields
= int(catheaderdata
[1], 16);
4649 catftype
= int(catheaderdata
[2], 16);
4650 if(re
.findall(r
"^[.|/]", catheaderdata
[3])):
4651 catfname
= catheaderdata
[3];
4653 catfname
= "./"+catheaderdata
[3];
4654 catfbasedir
= os
.path
.dirname(catfname
);
4655 catflinkname
= catheaderdata
[4];
4656 catfsize
= int(catheaderdata
[5], 16);
4657 catfatime
= int(catheaderdata
[6], 16);
4658 catfmtime
= int(catheaderdata
[7], 16);
4659 catfctime
= int(catheaderdata
[8], 16);
4660 catfbtime
= int(catheaderdata
[9], 16);
4661 catfmode
= int(catheaderdata
[10], 16);
4662 catfchmode
= stat
.S_IMODE(catfmode
);
4663 catftypemod
= stat
.S_IFMT(catfmode
);
4664 catfwinattributes
= int(catheaderdata
[11], 16);
4665 catfcompression
= catheaderdata
[12];
4666 catfcsize
= int(catheaderdata
[13], 16);
4667 catfuid
= int(catheaderdata
[14], 16);
4668 catfuname
= catheaderdata
[15];
4669 catfgid
= int(catheaderdata
[16], 16);
4670 catfgname
= catheaderdata
[17];
4671 fid
= int(catheaderdata
[18], 16);
4672 finode
= int(catheaderdata
[19], 16);
4673 flinkcount
= int(catheaderdata
[20], 16);
4674 catfdev_minor
= int(catheaderdata
[21], 16);
4675 catfdev_major
= int(catheaderdata
[22], 16);
4676 catfrdev_minor
= int(catheaderdata
[23], 16);
4677 catfrdev_major
= int(catheaderdata
[24], 16);
4678 catfseeknextfile
= catheaderdata
[25];
4679 catfextrasize
= int(catheaderdata
[26], 16);
4680 catfextrafields
= int(catheaderdata
[27], 16);
4681 extrafieldslist
= [];
4683 extraend
= extrastart
+ catfextrafields
;
4684 extrafieldslist
= [];
4685 if(extrastart
<extraend
):
4686 extrafieldslist
.append(catheaderdata
[extrastart
]);
4687 extrastart
= extrastart
+ 1;
4688 catfcs
= catheaderdata
[extrastart
+ 2].lower();
4689 catfccs
= catheaderdata
[extrastart
+ 3].lower();
4690 catnewfcs
= GetHeaderChecksum(catheaderdata
[:-2], catheaderdata
[-4].lower(), True, formatspecs
);
4692 VerbosePrintOut(catfname
);
4693 VerbosePrintOut("Record Number " + str(il
) + "; File ID " + str(fid
) + "; iNode Number " + str(finode
));
4694 if(catfcs
==catnewfcs
):
4696 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart
));
4697 VerbosePrintOut("'" + str(catfcs
) + "' == " + "'" + str(catnewfcs
) + "'");
4700 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart
));
4701 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4702 valid_archive
= False;
4703 invalid_archive
= True;
4704 catfhend
= catfp
.tell() - 1;
4705 catfcontentstart
= catfp
.tell();
4707 pyhascontents
= False;
4709 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4710 catfcontents
= catfp
.read(catfsize
);
4712 catfcontents
= catfp
.read(catfcsize
);
4713 catnewfccs
= GetFileChecksum(catfcontents
, catheaderdata
[-3].lower(), False, formatspecs
);
4714 pyhascontents
= True;
4715 if(catfccs
==catnewfccs
):
4717 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart
));
4718 VerbosePrintOut("'" + str(catfccs
) + "' == " + "'" + str(catnewfccs
) + "'");
4721 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart
));
4722 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4723 valid_archive
= False;
4724 invalid_archive
= True;
4726 VerbosePrintOut("");
4727 if(re
.findall(r
"^\+([0-9]+)", catfseeknextfile
)):
4728 fseeknextasnum
= int(catfseeknextfile
.replace("+", ""));
4729 if(abs(fseeknextasnum
)==0):
4731 catfp
.seek(fseeknextasnum
, 1);
4732 elif(re
.findall(r
"^\-([0-9]+)", catfseeknextfile
)):
4733 fseeknextasnum
= int(catfseeknextfile
);
4734 if(abs(fseeknextasnum
)==0):
4736 catfp
.seek(fseeknextasnum
, 1);
4737 elif(re
.findall(r
"^([0-9]+)", catfseeknextfile
)):
4738 fseeknextasnum
= int(catfseeknextfile
);
4739 if(abs(fseeknextasnum
)==0):
4741 catfp
.seek(fseeknextasnum
, 0);
4755 create_alias_function("", __file_format_name__
, "Validate", ArchiveFileValidate
);
4757 def ArchiveFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4758 formatspecs
= FormatSpecsListToDict(formatspecs
);
4759 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4762 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4763 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4764 if(checkcompressfile
=="tarfile"):
4765 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4766 if(checkcompressfile
=="zipfile"):
4767 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4768 if(rarfile_support
and checkcompressfile
=="rarfile"):
4769 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4770 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4771 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4772 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4779 if(hasattr(sys
.stdin
, "buffer")):
4780 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4782 shutil
.copyfileobj(sys
.stdin
, catfp
);
4784 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4788 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4789 catfp
= download_file_from_internet_file(infile
);
4790 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4796 infile
= RemoveWindowsPath(infile
);
4797 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4798 if(checkcompressfile
=="tarfile"):
4799 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4800 if(checkcompressfile
=="zipfile"):
4801 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4802 if(rarfile_support
and checkcompressfile
=="rarfile"):
4803 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4804 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4805 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4806 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4808 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4809 if(not compresscheck
):
4810 fextname
= os
.path
.splitext(infile
)[1];
4811 if(fextname
==".gz"):
4812 compresscheck
= "gzip";
4813 elif(fextname
==".bz2"):
4814 compresscheck
= "bzip2";
4815 elif(fextname
==".zst"):
4816 compresscheck
= "zstd";
4817 elif(fextname
==".lz4" or fextname
==".clz4"):
4818 compresscheck
= "lz4";
4819 elif(fextname
==".lzo" or fextname
==".lzop"):
4820 compresscheck
= "lzo";
4821 elif(fextname
==".lzma" or fextname
==".xz"):
4822 compresscheck
= "lzma";
4825 if(not compresscheck
):
4827 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4832 SeekToEndOfFile(catfp);
4834 SeekToEndOfFile(catfp);
4835 CatSize = catfp.tell();
4836 CatSizeEnd = CatSize;
4844 curloc
= catfp
.tell();
4847 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4849 catfp
.seek(curloc
, 0);
4850 catstring
= catheader
[0];
4851 catversion
= re
.findall(r
"([\d]+)", catstring
);
4852 fprenumfiles
= catheader
[1];
4853 fnumfiles
= int(fprenumfiles
, 16);
4854 fprechecksumtype
= catheader
[2];
4855 fprechecksum
= catheader
[3];
4856 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4857 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4858 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4859 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4860 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4861 fheadtell
= len(fileheader
);
4862 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4863 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4864 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4866 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4867 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': []};
4868 if(seekstart
<0 and seekstart
>fnumfiles
):
4870 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
4871 seekend
= fnumfiles
;
4872 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
4873 seekend
= fnumfiles
- abs(seekend
);
4876 while(il
< seekstart
):
4877 prefhstart
= catfp
.tell();
4878 if(formatspecs
['new_style']):
4879 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4881 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4882 if(len(preheaderdata
)==0):
4884 prefheadsize
= int(preheaderdata
[0], 16);
4885 prefnumfields
= int(preheaderdata
[1], 16);
4886 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4887 prefname
= preheaderdata
[3];
4889 prefname
= "./"+preheaderdata
[3];
4890 prefsize
= int(preheaderdata
[5], 16);
4891 prefcompression
= preheaderdata
[12];
4892 prefcsize
= int(preheaderdata
[13], 16);
4893 prefseeknextfile
= preheaderdata
[25];
4894 prefextrasize
= int(preheaderdata
[26], 16);
4895 prefextrafields
= int(preheaderdata
[27], 16);
4896 extrafieldslist
= [];
4898 extraend
= extrastart
+ prefextrafields
;
4899 extrafieldslist
= [];
4900 if(extrastart
<extraend
):
4901 extrafieldslist
.append(preheaderdata
[extrastart
]);
4902 extrastart
= extrastart
+ 1;
4903 prefcs
= preheaderdata
[extrastart
+ 2].lower();
4904 prefccs
= preheaderdata
[extrastart
+ 3].lower();
4905 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
4906 if(prefcs
!=prenewfcs
and not skipchecksum
):
4907 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4908 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4910 valid_archive
= False;
4911 invalid_archive
= True;
4912 prefhend
= catfp
.tell() - 1;
4913 prefcontentstart
= catfp
.tell();
4915 pyhascontents
= False;
4917 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
4918 prefcontents
= catfp
.read(prefsize
);
4920 prefcontents
= catfp
.read(prefcsize
);
4921 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4922 pyhascontents
= True;
4923 if(prefccs
!=prenewfccs
and not skipchecksum
):
4924 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4925 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4927 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
4928 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
4929 if(abs(fseeknextasnum
)==0):
4931 catfp
.seek(fseeknextasnum
, 1);
4932 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
4933 fseeknextasnum
= int(prefseeknextfile
);
4934 if(abs(fseeknextasnum
)==0):
4936 catfp
.seek(fseeknextasnum
, 1);
4937 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
4938 fseeknextasnum
= int(prefseeknextfile
);
4939 if(abs(fseeknextasnum
)==0):
4941 catfp
.seek(fseeknextasnum
, 0);
4945 fileidnum
= seekstart
;
4947 while(fileidnum
<seekend
):
4948 catfhstart
= catfp
.tell();
4949 if(formatspecs
['new_style']):
4950 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4952 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4953 if(len(catheaderdata
)==0):
4955 catfheadsize
= int(catheaderdata
[0], 16);
4956 catfnumfields
= int(catheaderdata
[1], 16);
4957 catftype
= int(catheaderdata
[2], 16);
4958 if(re
.findall(r
"^[.|/]", catheaderdata
[3])):
4959 catfname
= catheaderdata
[3];
4961 catfname
= "./"+catheaderdata
[3];
4962 catfbasedir
= os
.path
.dirname(catfname
);
4963 catflinkname
= catheaderdata
[4];
4964 catfsize
= int(catheaderdata
[5], 16);
4965 catfatime
= int(catheaderdata
[6], 16);
4966 catfmtime
= int(catheaderdata
[7], 16);
4967 catfctime
= int(catheaderdata
[8], 16);
4968 catfbtime
= int(catheaderdata
[9], 16);
4969 catfmode
= int(catheaderdata
[10], 16);
4970 catfchmode
= stat
.S_IMODE(catfmode
);
4971 catftypemod
= stat
.S_IFMT(catfmode
);
4972 catfwinattributes
= int(catheaderdata
[11], 16);
4973 catfcompression
= catheaderdata
[12];
4974 catfcsize
= int(catheaderdata
[13], 16);
4975 catfuid
= int(catheaderdata
[14], 16);
4976 catfuname
= catheaderdata
[15];
4977 catfgid
= int(catheaderdata
[16], 16);
4978 catfgname
= catheaderdata
[17];
4979 catfid
= int(catheaderdata
[18], 16);
4980 catfinode
= int(catheaderdata
[19], 16);
4981 catflinkcount
= int(catheaderdata
[20], 16);
4982 catfdev_minor
= int(catheaderdata
[21], 16);
4983 catfdev_major
= int(catheaderdata
[22], 16);
4984 catfrdev_minor
= int(catheaderdata
[23], 16);
4985 catfrdev_major
= int(catheaderdata
[24], 16);
4986 catfseeknextfile
= catheaderdata
[25];
4987 catfextrasize
= int(catheaderdata
[26], 16);
4988 catfextrafields
= int(catheaderdata
[27], 16);
4989 extrafieldslist
= [];
4991 extraend
= extrastart
+ catfextrafields
;
4992 extrafieldslist
= [];
4993 if(extrastart
<extraend
):
4994 extrafieldslist
.append(catheaderdata
[extrastart
]);
4995 extrastart
= extrastart
+ 1;
4996 catfcs
= catheaderdata
[extrastart
+ 2].lower();
4997 catfccs
= catheaderdata
[extrastart
+ 3].lower();
4998 catnewfcs
= GetHeaderChecksum(catheaderdata
[:-2], catheaderdata
[-4].lower(), True, formatspecs
);
4999 if(catfcs
!=catnewfcs
and not skipchecksum
):
5000 VerbosePrintOut("File Header Checksum Error with file " + catfname
+ " at offset " + str(catfhstart
));
5001 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
5003 catfhend
= catfp
.tell() - 1;
5004 catfcontentstart
= catfp
.tell();
5005 catfcontents
= BytesIO();
5006 pyhascontents
= False;
5007 if(catfsize
>0 and not listonly
):
5008 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
5009 catfcontents
.write(catfp
.read(catfsize
));
5011 catfcontents
.write(catfp
.read(catfcsize
));
5012 catfcontents
.seek(0, 0);
5013 catnewfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
5014 pyhascontents
= True;
5015 if(catfccs
!=catnewfccs
and skipchecksum
):
5016 VerbosePrintOut("File Content Checksum Error with file " + catfname
+ " at offset " + str(catfcontentstart
));
5017 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
5019 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
5022 catfcontents
.seek(0, 0);
5024 catfcontents
= UncompressArchiveFile(catfcontents
, formatspecs
);
5025 catfcontents
.seek(0, 0);
5026 catfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
5027 if(catfsize
>0 and listonly
):
5028 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
5029 catfp
.seek(catfsize
, 1);
5031 catfp
.seek(catfcsize
, 1);
5032 pyhascontents
= False;
5033 catfcontentend
= catfp
.tell();
5034 if(re
.findall(r
"^\+([0-9]+)", catfseeknextfile
)):
5035 fseeknextasnum
= int(catfseeknextfile
.replace("+", ""));
5036 if(abs(fseeknextasnum
)==0):
5038 catfp
.seek(fseeknextasnum
, 1);
5039 elif(re
.findall(r
"^\-([0-9]+)", catfseeknextfile
)):
5040 fseeknextasnum
= int(catfseeknextfile
);
5041 if(abs(fseeknextasnum
)==0):
5043 catfp
.seek(fseeknextasnum
, 1);
5044 elif(re
.findall(r
"^([0-9]+)", catfseeknextfile
)):
5045 fseeknextasnum
= int(catfseeknextfile
);
5046 if(abs(fseeknextasnum
)==0):
5048 catfp
.seek(fseeknextasnum
, 0);
5051 catfcontents
.seek(0, 0);
5052 catlist
['ffilelist'].append({'fid': realidnum
, 'fidalt': fileidnum
, 'fheadersize': catfheadsize
, 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
, 'fatime': catfatime
, 'fmtime': catfmtime
, 'fctime': catfctime
, 'fbtime': catfbtime
, 'fmode': catfmode
, 'fchmode': catfchmode
, 'ftypemod': catftypemod
, 'fwinattributes': catfwinattributes
, 'fcompression': catfcompression
, 'fcsize': catfcsize
, 'fuid': catfuid
, 'funame': catfuname
, 'fgid': catfgid
, 'fgname': catfgname
, 'finode': catfinode
, 'flinkcount': catflinkcount
, 'fminor': catfdev_minor
, 'fmajor': catfdev_major
, 'frminor': catfrdev_minor
, 'frmajor': catfrdev_major
, 'fseeknextfile': catfseeknextfile
, 'fheaderchecksumtype': catheaderdata
[-4], 'fcontentchecksumtype': catheaderdata
[-3], 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': catfextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': catfcs
, 'fcontentchecksum': catfccs
, 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': catfcontents
});
5053 fileidnum
= fileidnum
+ 1;
5054 realidnum
= realidnum
+ 1;
5056 catlist
.update({'catfp': catfp
});
5061 create_alias_function("", __file_format_name__
, "ToArray", ArchiveFileToArray
);
5063 def ArchiveFileStringToArray(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5064 formatspecs
= FormatSpecsListToDict(formatspecs
);
5065 catfp
= BytesIO(catstr
);
5066 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5067 return listcatfiles
;
5069 create_alias_function("", __file_format_name__
, "StringToArray", ArchiveFileStringToArray
);
5071 def TarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5072 formatspecs
= FormatSpecsListToDict(formatspecs
);
5074 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5075 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5076 return listcatfiles
;
5078 def ZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5079 formatspecs
= FormatSpecsListToDict(formatspecs
);
5081 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5082 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5083 return listcatfiles
;
5085 if(not rarfile_support
):
5086 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5089 if(rarfile_support
):
5090 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5091 formatspecs
= FormatSpecsListToDict(formatspecs
);
5093 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5094 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5095 return listcatfiles
;
5097 if(not py7zr_support
):
5098 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5102 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5103 formatspecs
= FormatSpecsListToDict(formatspecs
);
5105 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5106 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5107 return listcatfiles
;
5109 def InFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5110 formatspecs
= FormatSpecsListToDict(formatspecs
);
5111 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
5112 if(checkcompressfile
=="tarfile"):
5113 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5114 elif(checkcompressfile
=="zipfile"):
5115 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5116 elif(checkcompressfile
=="catfile"):
5117 return ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5118 elif(rarfile_support
and checkcompressfile
=="rarfile"):
5119 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5120 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
5121 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5126 def ListDirToArrayAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5127 formatspecs
= FormatSpecsListToDict(formatspecs
);
5128 catver
= formatspecs
['format_ver'];
5129 fileheaderver
= str(int(catver
.replace(".", "")));
5130 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5131 advancedlist
= formatspecs
['use_advanced_list'];
5132 altinode
= formatspecs
['use_alt_inode'];
5135 for line
in sys
.stdin
:
5136 infilelist
.append(line
.strip());
5137 infilelist
= list(filter(None, infilelist
));
5138 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
5139 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
5141 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
5142 for line
in finfile
:
5143 infilelist
.append(line
.strip());
5144 infilelist
= list(filter(None, infilelist
));
5146 if(isinstance(infiles
, (list, tuple, ))):
5147 infilelist
= list(filter(None, infiles
));
5148 elif(isinstance(infiles
, (str, ))):
5149 infilelist
= list(filter(None, [infiles
]));
5151 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
5153 GetDirList
= ListDir(infilelist
, followlink
, False);
5161 inodetocatinode
= {};
5163 fnumfiles
= int(len(GetDirList
));
5164 catver
= formatspecs
['format_ver'];
5165 fileheaderver
= str(int(catver
.replace(".", "")));
5166 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5167 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5168 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5169 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5170 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5171 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5172 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5173 fheadtell
= len(fileheader
);
5174 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5175 for curfname
in GetDirList
:
5176 catfhstart
= fheadtell
;
5177 if(re
.findall(r
"^[.|/]", curfname
)):
5180 fname
= "./"+curfname
;
5182 VerbosePrintOut(fname
);
5183 if(not followlink
or followlink
is None):
5184 fstatinfo
= os
.lstat(fname
);
5186 fstatinfo
= os
.stat(fname
);
5187 fpremode
= fstatinfo
.st_mode
;
5188 finode
= fstatinfo
.st_ino
;
5189 flinkcount
= fstatinfo
.st_nlink
;
5191 if(stat
.S_ISREG(fpremode
)):
5193 elif(stat
.S_ISLNK(fpremode
)):
5195 elif(stat
.S_ISCHR(fpremode
)):
5197 elif(stat
.S_ISBLK(fpremode
)):
5199 elif(stat
.S_ISDIR(fpremode
)):
5201 elif(stat
.S_ISFIFO(fpremode
)):
5203 elif(stat
.S_ISSOCK(fpremode
)):
5205 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
5207 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
5209 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
5214 fbasedir
= os
.path
.dirname(fname
);
5216 if(not followlink
and finode
!=0):
5218 if(finode
in inodelist
):
5220 flinkname
= inodetofile
[finode
];
5224 fcurinode
= inodetocatinode
[finode
];
5225 if(finode
not in inodelist
):
5226 inodelist
.append(finode
);
5227 inodetofile
.update({finode
: fname
});
5228 inodetocatinode
.update({finode
: curinode
});
5232 fcurinode
= curinode
;
5233 curinode
= curinode
+ 1;
5235 fcurinode
= curinode
;
5236 curinode
= curinode
+ 1;
5237 curfid
= curfid
+ 1;
5239 flinkname
= os
.readlink(fname
);
5240 fdev
= fstatinfo
.st_dev
;
5241 getfdev
= GetDevMajorMinor(fdev
);
5242 fdev_minor
= getfdev
[0];
5243 fdev_major
= getfdev
[1];
5244 frdev
= fstatinfo
.st_dev
;
5245 if(hasattr(fstatinfo
, "st_rdev")):
5246 frdev
= fstatinfo
.st_rdev
;
5248 frdev
= fstatinfo
.st_dev
;
5249 getfrdev
= GetDevMajorMinor(frdev
);
5250 frdev_minor
= getfrdev
[0];
5251 frdev_major
= getfrdev
[1];
5252 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5254 if(ftype
==0 or ftype
==7):
5255 fsize
= fstatinfo
.st_size
;
5256 fatime
= fstatinfo
.st_atime
;
5257 fmtime
= fstatinfo
.st_mtime
;
5258 fctime
= fstatinfo
.st_ctime
;
5259 if(hasattr(fstatinfo
, "st_birthtime")):
5260 fbtime
= fstatinfo
.st_birthtime
;
5262 fbtime
= fstatinfo
.st_ctime
;
5263 fmode
= fstatinfo
.st_mode
;
5264 fchmode
= stat
.S_IMODE(fstatinfo
.st_mode
);
5265 ftypemod
= stat
.S_IFMT(fstatinfo
.st_mode
);
5266 fuid
= fstatinfo
.st_uid
;
5267 fgid
= fstatinfo
.st_gid
;
5272 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
5273 funame
= userinfo
.pw_name
;
5282 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
5283 fgname
= groupinfo
.gr_name
;
5288 fdev_minor
= fdev_minor
;
5289 fdev_major
= fdev_major
;
5290 frdev_minor
= frdev_minor
;
5291 frdev_major
= frdev_major
;
5292 flinkcount
= flinkcount
;
5293 if(hasattr(fstatinfo
, "st_file_attributes")):
5294 fwinattributes
= fstatinfo
.st_file_attributes
;
5299 fcontents
= BytesIO();
5300 if(ftype
==0 or ftype
==7):
5301 with
open(fname
, "rb") as fpc
:
5302 shutil
.copyfileobj(fpc
, fcontents
);
5303 if(followlink
and (ftype
==1 or ftype
==2)):
5304 flstatinfo
= os
.stat(flinkname
);
5305 with
open(flinkname
, "rb") as fpc
:
5306 shutil
.copyfileobj(fpc
, fcontents
);
5307 fcontents
.seek(0, 0);
5308 ftypehex
= format(ftype
, 'x').lower();
5309 extrafields
= len(extradata
);
5310 extrafieldslist
= extradata
;
5311 catfextrafields
= extrafields
;
5312 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5313 if(len(extradata
)>0):
5314 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5315 extrasizelen
= len(extrasizestr
);
5316 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5317 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5318 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5319 catoutlenhex
= format(catoutlen
, 'x').lower();
5320 catoutlist
.insert(0, catoutlenhex
);
5321 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5322 catheaderdata
= catoutlist
;
5323 if(len(extradata
)>0):
5324 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5325 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5326 catfnumfields
= catoutlen
;
5327 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5328 fcontents
.seek(0, 0);
5329 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5330 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5331 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5332 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5333 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5334 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5335 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5336 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5337 catfcontentstart
= fheadtell
;
5338 fheadtell
+= len(catfileoutstr
) + 1;
5339 catfcontentend
= fheadtell
- 1;
5340 catfhend
= catfcontentend
;
5341 fcontents
.seek(0, 0);
5342 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5343 pyhascontents
= False;
5344 if(int(fsize
)>0 and not listonly
):
5345 pyhascontents
= True;
5346 if(int(fsize
)>0 and listonly
):
5347 fcontents
= BytesIO();
5348 pyhascontents
= False;
5349 fcontents
.seek(0, 0);
5350 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5351 fileidnum
= fileidnum
+ 1;
5354 def TarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5355 formatspecs
= FormatSpecsListToDict(formatspecs
);
5361 inodetocatinode
= {};
5365 if(hasattr(sys
.stdin
, "buffer")):
5366 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5368 shutil
.copyfileobj(sys
.stdin
, infile
);
5373 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5374 infile
= download_file_from_internet_file(infile
);
5379 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5381 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
5383 if(not tarfile
.TarFileCheck(infile
)):
5385 except AttributeError:
5386 if(not TarFileCheck(infile
)):
5391 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
5392 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
5394 tarfp
= tarfile
.open(infile
, "r");
5395 except FileNotFoundError
:
5397 fnumfiles
= int(len(tarfp
.getmembers()));
5398 catver
= formatspecs
['format_ver'];
5399 fileheaderver
= str(int(catver
.replace(".", "")));
5400 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5401 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5402 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5403 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5404 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5405 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5406 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5407 fheadtell
= len(fileheader
);
5408 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5409 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
5410 catfhstart
= fheadtell
;
5411 if(re
.findall(r
"^[.|/]", member
.name
)):
5412 fname
= member
.name
;
5414 fname
= "./"+member
.name
;
5416 VerbosePrintOut(fname
);
5417 fpremode
= member
.mode
;
5418 ffullmode
= member
.mode
;
5422 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5424 elif(member
.isdev()):
5425 ffullmode
= member
.mode
;
5427 elif(member
.islnk()):
5428 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5430 elif(member
.issym()):
5431 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
5433 elif(member
.ischr()):
5434 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
5436 elif(member
.isblk()):
5437 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
5439 elif(member
.isdir()):
5440 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
5442 elif(member
.isfifo()):
5443 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
5445 elif(member
.issparse()):
5446 ffullmode
= member
.mode
;
5449 ffullmode
= member
.mode
;
5452 fbasedir
= os
.path
.dirname(fname
);
5456 curfid
= curfid
+ 1;
5458 flinkname
= member
.linkname
;
5459 fdev_minor
= member
.devminor
;
5460 fdev_major
= member
.devmajor
;
5461 frdev_minor
= member
.devminor
;
5462 frdev_major
= member
.devmajor
;
5463 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5465 elif(ftype
==0 or ftype
==7):
5466 fsize
= member
.size
;
5468 fsize
= member
.size
;
5469 fatime
= member
.mtime
;
5470 fmtime
= member
.mtime
;
5471 fctime
= member
.mtime
;
5472 fbtime
= member
.mtime
;
5474 fchmode
= stat
.S_IMODE(ffullmode
);
5475 ftypemod
= stat
.S_IFMT(ffullmode
);
5478 funame
= member
.uname
;
5479 fgname
= member
.gname
;
5480 flinkcount
= flinkcount
;
5481 fwinattributes
= int(0);
5484 fcontents
= BytesIO();
5485 if(ftype
==0 or ftype
==7):
5486 with tarfp
.extractfile(member
) as fpc
:
5487 shutil
.copyfileobj(fpc
, fcontents
);
5488 fcontents
.seek(0, 0);
5489 ftypehex
= format(ftype
, 'x').lower();
5490 extrafields
= len(extradata
);
5491 extrafieldslist
= extradata
;
5492 catfextrafields
= extrafields
;
5493 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5494 if(len(extradata
)>0):
5495 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5496 extrasizelen
= len(extrasizestr
);
5497 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5498 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5499 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5500 catoutlenhex
= format(catoutlen
, 'x').lower();
5501 catoutlist
.insert(0, catoutlenhex
);
5502 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5503 catheaderdata
= catoutlist
;
5504 if(len(extradata
)>0):
5505 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5506 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5507 catfnumfields
= catoutlen
;
5508 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5509 fcontents
.seek(0, 0);
5510 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5511 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5512 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5513 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5514 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5515 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5516 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5517 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5518 catfcontentstart
= fheadtell
;
5519 fheadtell
+= len(catfileoutstr
) + 1;
5520 catfcontentend
= fheadtell
- 1;
5521 catfhend
= catfcontentend
;
5522 fcontents
.seek(0, 0);
5523 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5524 pyhascontents
= False;
5525 if(int(fsize
)>0 and not listonly
):
5526 pyhascontents
= True;
5527 if(int(fsize
)>0 and listonly
):
5528 fcontents
= BytesIO();
5529 pyhascontents
= False;
5530 fcontents
.seek(0, 0);
5531 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5532 fileidnum
= fileidnum
+ 1;
5535 def ZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5536 formatspecs
= FormatSpecsListToDict(formatspecs
);
5542 inodetocatinode
= {};
5546 if(hasattr(sys
.stdin
, "buffer")):
5547 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5549 shutil
.copyfileobj(sys
.stdin
, infile
);
5554 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5555 infile
= download_file_from_internet_file(infile
);
5560 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5564 if(not zipfile
.is_zipfile(infile
)):
5567 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
5568 except FileNotFoundError
:
5570 ziptest
= zipfp
.testzip();
5572 VerbosePrintOut("Bad file found!");
5573 fnumfiles
= int(len(zipfp
.infolist()));
5574 catver
= formatspecs
['format_ver'];
5575 fileheaderver
= str(int(catver
.replace(".", "")));
5576 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5577 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5578 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5579 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5580 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5581 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5582 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5583 fheadtell
= len(fileheader
);
5584 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5585 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
5586 catfhstart
= fheadtell
;
5587 if(re
.findall(r
"^[.|/]", member
.filename
)):
5588 fname
= member
.filename
;
5590 fname
= "./"+member
.filename
;
5591 zipinfo
= zipfp
.getinfo(member
.filename
);
5593 VerbosePrintOut(fname
);
5594 if(not member
.is_dir()):
5595 fpremode
= stat
.S_IFREG
+ 438;
5596 elif(member
.is_dir()):
5597 fpremode
= stat
.S_IFDIR
+ 511;
5600 if(not member
.is_dir()):
5602 elif(member
.is_dir()):
5605 fbasedir
= os
.path
.dirname(fname
);
5609 curfid
= curfid
+ 1;
5617 fsize
= member
.file_size
;
5619 fsize
= member
.file_size
;
5620 fatime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5621 fmtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5622 fctime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5623 fbtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5624 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
5625 fwinattributes
= int(zipinfo
.external_attr
);
5626 if(not member
.is_dir()):
5627 fmode
= int(stat
.S_IFREG
+ 438);
5628 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5629 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5630 elif(member
.is_dir()):
5631 fmode
= int(stat
.S_IFDIR
+ 511);
5632 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5633 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5634 elif(zipinfo
.create_system
==3):
5635 fwinattributes
= int(0);
5636 fmode
= int(zipinfo
.external_attr
);
5638 fwinattributes
= int(0);
5639 if(not member
.is_dir()):
5640 fmode
= int(stat
.S_IFREG
+ 438);
5641 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5642 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5643 elif(member
.is_dir()):
5644 fmode
= int(stat
.S_IFDIR
+ 511);
5645 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5646 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5651 except AttributeError:
5657 except AttributeError:
5664 userinfo
= pwd
.getpwuid(os
.getuid());
5665 funame
= userinfo
.pw_name
;
5668 except AttributeError:
5676 groupinfo
= grp
.getgrgid(os
.getgid());
5677 fgname
= groupinfo
.gr_name
;
5680 except AttributeError:
5684 fcontents
= BytesIO();
5686 fcontents
.write(zipfp
.read(member
.filename
));
5687 fcontents
.seek(0, 0);
5688 ftypehex
= format(ftype
, 'x').lower();
5689 extrafields
= len(extradata
);
5690 extrafieldslist
= extradata
;
5691 catfextrafields
= extrafields
;
5692 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5693 if(len(extradata
)>0):
5694 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5695 extrasizelen
= len(extrasizestr
);
5696 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5697 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5698 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5699 catoutlenhex
= format(catoutlen
, 'x').lower();
5700 catoutlist
.insert(0, catoutlenhex
);
5701 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5702 catheaderdata
= catoutlist
;
5703 if(len(extradata
)>0):
5704 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5705 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5706 catfnumfields
= catoutlen
;
5707 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5708 fcontents
.seek(0, 0);
5709 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5710 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5711 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5712 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5713 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5714 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5715 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5716 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5717 catfcontentstart
= fheadtell
;
5718 fheadtell
+= len(catfileoutstr
) + 1;
5719 catfcontentend
= fheadtell
- 1;
5720 catfhend
= catfcontentend
;
5721 fcontents
.seek(0, 0);
5722 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5723 pyhascontents
= False;
5724 if(int(fsize
)>0 and not listonly
):
5725 pyhascontents
= True;
5726 if(int(fsize
)>0 and listonly
):
5727 fcontents
= BytesIO();
5728 pyhascontents
= False;
5729 fcontents
.seek(0, 0);
5730 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5731 fileidnum
= fileidnum
+ 1;
5734 if(not rarfile_support
):
5735 def RarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5738 if(rarfile_support
):
5739 def RarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5740 formatspecs
= FormatSpecsListToDict(formatspecs
);
5746 inodetocatinode
= {};
5748 if(not os
.path
.exists(infile
,) or not os
.path
.isfile(infile
,)):
5750 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
5752 rarfp
= rarfile
.RarFile(infile
, "r");
5753 rartest
= rarfp
.testrar();
5755 VerbosePrintOut("Bad file found!");
5756 fnumfiles
= int(len(rarfp
.infolist()));
5757 catver
= formatspecs
['format_ver'];
5758 fileheaderver
= str(int(catver
.replace(".", "")));
5759 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5760 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5761 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5762 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5763 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5764 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5765 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5766 fheadtell
= len(fileheader
);
5767 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5768 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
5769 catfhstart
= fheadtell
;
5772 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
5775 member
.external_attr
5777 except AttributeError:
5779 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
5782 member
.external_attr
5784 except AttributeError:
5789 if(re
.findall(r
"^[.|/]", member
.filename
)):
5790 fname
= member
.filename
;
5792 fname
= "./"+member
.filename
;
5793 rarinfo
= rarfp
.getinfo(member
.filename
);
5795 VerbosePrintOut(fname
);
5796 if(is_unix
and member
.external_attr
!=0):
5797 fpremode
= int(member
.external_attr
);
5798 elif(member
.is_file()):
5799 fpremode
= stat
.S_IFREG
+ 438;
5800 elif(member
.is_symlink()):
5801 fpremode
= stat
.S_IFLNK
+ 438;
5802 elif(member
.is_dir()):
5803 fpremode
= stat
.S_IFDIR
+ 511;
5804 if(is_windows
and member
.external_attr
!=0):
5805 fwinattributes
= int(member
.external_attr
);
5807 fwinattributes
= int(0);
5812 if(member
.is_file()):
5814 elif(member
.is_symlink()):
5816 elif(member
.is_dir()):
5820 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
5821 fbasedir
= os
.path
.dirname(fname
);
5825 curfid
= curfid
+ 1;
5833 fsize
= member
.file_size
;
5836 fatime
= int(member
.atime
.timestamp());
5838 fatime
= int(member
.mtime
.timestamp());
5839 except AttributeError:
5840 fatime
= int(member
.mtime
.timestamp());
5841 fmtime
= int(member
.mtime
.timestamp());
5844 fctime
= int(member
.ctime
.timestamp());
5846 fctime
= int(member
.mtime
.timestamp());
5847 except AttributeError:
5848 fctime
= int(member
.mtime
.timestamp());
5849 fbtime
= int(member
.mtime
.timestamp());
5850 if(is_unix
and member
.external_attr
!=0):
5851 fmode
= int(member
.external_attr
);
5852 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
5853 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
5854 elif(member
.is_file()):
5855 fmode
= int(stat
.S_IFREG
+ 438)
5856 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5857 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5858 elif(member
.is_symlink()):
5859 fmode
= int(stat
.S_IFLNK
+ 438)
5860 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5861 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5862 elif(member
.is_dir()):
5863 fmode
= int(stat
.S_IFDIR
+ 511)
5864 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5865 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5868 except AttributeError:
5874 except AttributeError:
5881 userinfo
= pwd
.getpwuid(os
.getuid());
5882 funame
= userinfo
.pw_name
;
5885 except AttributeError:
5893 groupinfo
= grp
.getgrgid(os
.getgid());
5894 fgname
= groupinfo
.gr_name
;
5897 except AttributeError:
5901 fcontents
= BytesIO();
5903 fcontents
.write(rarfp
.read(member
.filename
));
5904 fcontents
.seek(0, 0);
5905 ftypehex
= format(ftype
, 'x').lower();
5906 extrafields
= len(extradata
);
5907 extrafieldslist
= extradata
;
5908 catfextrafields
= extrafields
;
5909 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5910 if(len(extradata
)>0):
5911 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5912 extrasizelen
= len(extrasizestr
);
5913 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5914 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5915 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5916 catoutlenhex
= format(catoutlen
, 'x').lower();
5917 catoutlist
.insert(0, catoutlenhex
);
5918 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5919 if(len(extradata
)>0):
5920 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5921 ccatfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5922 catfnumfields
= 24 + catfextrafields
;
5923 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5924 fcontents
.seek(0, 0);
5925 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5926 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5927 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5928 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5929 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5930 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5931 catheaderdata
= catoutlist
;
5932 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5933 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5934 catfcontentstart
= fheadtell
;
5935 fheadtell
+= len(catfileoutstr
) + 1;
5936 catfcontentend
= fheadtell
- 1;
5937 catfhend
= catfcontentend
;
5938 fcontents
.seek(0, 0);
5939 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5940 pyhascontents
= False;
5941 if(int(fsize
)>0 and not listonly
):
5942 pyhascontents
= True;
5943 if(int(fsize
)>0 and listonly
):
5944 fcontents
= BytesIO();
5945 pyhascontents
= False;
5946 fcontents
.seek(0, 0);
5947 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5948 fileidnum
= fileidnum
+ 1;
5951 if(not py7zr_support
):
5952 def SevenZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5956 def SevenZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5957 formatspecs
= FormatSpecsListToDict(formatspecs
);
5963 inodetocatinode
= {};
5965 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
5966 file_content
= szpfp
.readall();
5967 #sztest = szpfp.testzip();
5968 sztestalt
= szpfp
.test();
5970 VerbosePrintOut("Bad file found!");
5971 numfiles
= int(len(szpfp
.list()));
5972 catver
= formatspecs
['format_ver'];
5973 fileheaderver
= str(int(catver
.replace(".", "")));
5974 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5975 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5976 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5977 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5978 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5979 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5980 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5981 fheadtell
= len(fileheader
);
5982 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5983 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
5984 catfhstart
= fheadtell
;
5985 if(re
.findall(r
"^[.|/]", member
.filename
)):
5986 fname
= member
.filename
;
5988 fname
= "./"+member
.filename
;
5989 if(not member
.is_directory
):
5990 fpremode
= int(stat
.S_IFREG
+ 438);
5991 elif(member
.is_directory
):
5992 fpremode
= int(stat
.S_IFDIR
+ 511);
5993 fwinattributes
= int(0);
5998 if(member
.is_directory
):
6003 fbasedir
= os
.path
.dirname(fname
);
6007 curfid
= curfid
+ 1;
6014 fatime
= int(member
.creationtime
.timestamp());
6015 fmtime
= int(member
.creationtime
.timestamp());
6016 fctime
= int(member
.creationtime
.timestamp());
6017 fbtime
= int(member
.creationtime
.timestamp());
6018 if(member
.is_directory
):
6019 fmode
= int(stat
.S_IFDIR
+ 511)
6020 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
6021 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
6023 fmode
= int(stat
.S_IFLNK
+ 438)
6024 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
6025 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
6028 except AttributeError:
6034 except AttributeError:
6041 userinfo
= pwd
.getpwuid(os
.getuid());
6042 funame
= userinfo
.pw_name
;
6045 except AttributeError:
6053 groupinfo
= grp
.getgrgid(os
.getgid());
6054 fgname
= groupinfo
.gr_name
;
6057 except AttributeError:
6061 fcontents
= BytesIO();
6063 fcontents
.write(file_content
[member
.filename
].read());
6064 fsize
= format(fcontents
.tell(), 'x').lower();
6066 fcontents
.seek(0, 0);
6067 ftypehex
= format(ftype
, 'x').lower();
6068 extrafields
= len(extradata
);
6069 extrafieldslist
= extradata
;
6070 catfextrafields
= extrafields
;
6071 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
6072 if(len(extradata
)>0):
6073 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
6074 extrasizelen
= len(extrasizestr
);
6075 extrasizelenhex
= format(extrasizelen
, 'x').lower();
6076 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
6077 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
6078 catoutlenhex
= format(catoutlen
, 'x').lower();
6079 catoutlist
.insert(0, catoutlenhex
);
6080 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
6081 catheaderdata
= catoutlist
;
6082 if(len(extradata
)>0):
6083 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
6084 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
6085 catfnumfields
= 24 + catfextrafields
;
6086 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6087 fcontents
.seek(0, 0);
6088 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
6089 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
6090 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
6091 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
6092 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6093 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
6094 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
6095 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
6096 catfcontentstart
= fheadtell
;
6097 fheadtell
+= len(catfileoutstr
) + 1;
6098 catfcontentend
= fheadtell
- 1;
6099 catfhend
= catfcontentend
;
6100 fcontents
.seek(0, 0);
6101 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
6102 pyhascontents
= False;
6103 if(int(fsize
)>0 and not listonly
):
6104 pyhascontents
= True;
6105 if(int(fsize
)>0 and listonly
):
6106 fcontents
= BytesIO();
6107 pyhascontents
= False;
6108 fcontents
.seek(0, 0);
6109 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
6110 fileidnum
= fileidnum
+ 1;
6113 def InFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6114 formatspecs
= FormatSpecsListToDict(formatspecs
);
6115 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
6116 if(checkcompressfile
=="tarfile"):
6117 return TarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6118 elif(checkcompressfile
=="zipfile"):
6119 return ZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6120 elif(checkcompressfile
=="catfile"):
6121 return ArchiveFileToArray(infile
, 0, 0, listonly
, True, False, formatspecs
, False);
6122 elif(rarfile_support
and checkcompressfile
=="rarfile"):
6123 return RarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6124 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
6125 return SevenZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6130 def ListDirToArray(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6131 formatspecs
= FormatSpecsListToDict(formatspecs
);
6132 outarray
= BytesIO();
6133 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6134 listcatfiles
= ArchiveFileToArray(outarray
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6135 return listcatfiles
;
6137 def ArchiveFileArrayToArrayIndex(inarray
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6138 formatspecs
= FormatSpecsListToDict(formatspecs
);
6139 if(isinstance(inarray
, dict)):
6140 listcatfiles
= inarray
;
6143 if(not listcatfiles
):
6145 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6147 catarray
.update({'catfp': listcatfiles
['catfp']});
6148 lenlist
= len(listcatfiles
['ffilelist']);
6150 lcfx
= int(listcatfiles
['fnumfiles']);
6151 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6152 lcfx
= int(lenlist
);
6154 lcfx
= int(listcatfiles
['fnumfiles']);
6156 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6157 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6158 catarray
['filetoid'].update(filetoidarray
);
6159 catarray
['idtofile'].update(idtofilearray
);
6160 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6161 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6162 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6163 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6164 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6165 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6166 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6167 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6168 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6169 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6170 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6171 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6172 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6173 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6174 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6175 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6176 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6177 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6178 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6179 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6180 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6181 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6182 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6183 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6184 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6185 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6186 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6187 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6188 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6189 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6190 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6194 create_alias_function("", __file_format_name__
, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex
);
6196 def RePackArchiveFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6197 formatspecs
= FormatSpecsListToDict(formatspecs
);
6198 if(isinstance(infile
, dict)):
6199 listcatfiles
= infile
;
6201 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6202 infile
= RemoveWindowsPath(infile
);
6203 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6204 if(outfile
!="-" and not hasattr(infile
, "read") and not hasattr(outfile
, "write")):
6205 outfile
= RemoveWindowsPath(outfile
);
6206 checksumtype
= checksumtype
.lower();
6207 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
6208 checksumtype
="crc32";
6209 if(checksumtype
=="none"):
6211 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
6212 compression
= "auto";
6213 if(compression
not in compressionlist
and compression
is None):
6214 compression
= "auto";
6216 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6217 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
6218 if(os
.path
.exists(outfile
)):
6221 except OSError as e
:
6223 if(not listcatfiles
):
6228 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6230 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6233 fbasename
= os
.path
.splitext(outfile
)[0];
6234 fextname
= os
.path
.splitext(outfile
)[1];
6235 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
6236 catver
= formatspecs
['format_ver'];
6237 fileheaderver
= str(int(catver
.replace(".", "")));
6238 lenlist
= len(listcatfiles
['ffilelist']);
6239 fnumfiles
= int(listcatfiles
['fnumfiles']);
6240 if(lenlist
>fnumfiles
or lenlist
<fnumfiles
):
6241 fnumfiles
= lenlist
;
6242 AppendFileHeader(catfp
, fnumfiles
, checksumtype
, formatspecs
);
6243 lenlist
= len(listcatfiles
['ffilelist']);
6244 fnumfiles
= int(listcatfiles
['fnumfiles']);
6246 lcfx
= int(listcatfiles
['fnumfiles']);
6247 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6248 lcfx
= int(lenlist
);
6250 lcfx
= int(listcatfiles
['fnumfiles']);
6258 if(re
.findall(r
"^[.|/]", listcatfiles
['ffilelist'][reallcfi
]['fname'])):
6259 fname
= listcatfiles
['ffilelist'][reallcfi
]['fname'];
6261 fname
= "./"+listcatfiles
['ffilelist'][reallcfi
]['fname'];
6263 VerbosePrintOut(fname
);
6264 fheadersize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fheadersize']), 'x').lower();
6265 fsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fsize']), 'x').lower();
6266 flinkname
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6267 fatime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fatime']), 'x').lower();
6268 fmtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmtime']), 'x').lower();
6269 fctime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fctime']), 'x').lower();
6270 fbtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fbtime']), 'x').lower();
6271 fmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmode']), 'x').lower();
6272 fchmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fchmode']), 'x').lower();
6273 fuid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fuid']), 'x').lower();
6274 funame
= listcatfiles
['ffilelist'][reallcfi
]['funame'];
6275 fgid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fgid']), 'x').lower();
6276 fgname
= listcatfiles
['ffilelist'][reallcfi
]['fgname'];
6277 finode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['finode']), 'x').lower();
6278 flinkcount
= format(int(listcatfiles
['ffilelist'][reallcfi
]['flinkcount']), 'x').lower();
6279 fwinattributes
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fwinattributes']), 'x').lower();
6280 fcompression
= listcatfiles
['ffilelist'][reallcfi
]['fcompression'];
6281 fcsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fcsize']), 'x').lower();
6282 fdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fminor']), 'x').lower();
6283 fdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmajor']), 'x').lower();
6284 frdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frminor']), 'x').lower();
6285 frdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frmajor']), 'x').lower();
6286 fseeknextfile
= listcatfiles
['ffilelist'][reallcfi
]['fseeknextfile'];
6287 if(len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] and len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>0):
6288 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6289 if(not followlink
and len(extradata
)<0):
6290 extradata
= listcatfiles
['ffilelist'][reallcfi
]['fextralist'];
6291 fcontents
= listcatfiles
['ffilelist'][reallcfi
]['fcontents'];
6293 fcsize
= format(int(0), 'x').lower();
6294 if(not compresswholefile
):
6295 fcontents
.seek(0, 2);
6296 ucfsize
= fcontents
.tell();
6297 fcontents
.seek(0, 0);
6298 if(compression
=="auto"):
6299 ilsize
= len(compressionlistalt
);
6302 while(ilmin
< ilsize
):
6303 cfcontents
= BytesIO();
6304 shutil
.copyfileobj(fcontents
, cfcontents
);
6305 fcontents
.seek(0, 0);
6306 cfcontents
.seek(0, 0);
6307 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
6309 cfcontents
.seek(0, 2);
6310 ilcsize
.append(cfcontents
.tell());
6314 ilcsize
.append(sys
.maxint
);
6315 except AttributeError:
6316 ilcsize
.append(sys
.maxsize
);
6318 ilcmin
= ilcsize
.index(min(ilcsize
));
6319 compression
= compressionlistalt
[ilcmin
];
6320 fcontents
.seek(0, 0);
6321 cfcontents
= BytesIO();
6322 shutil
.copyfileobj(fcontents
, cfcontents
);
6323 cfcontents
.seek(0, 0);
6324 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
6325 cfcontents
.seek(0, 2);
6326 cfsize
= cfcontents
.tell();
6327 if(ucfsize
> cfsize
):
6328 fcsize
= format(int(cfsize
), 'x').lower();
6329 fcompression
= compression
;
6331 fcontents
= cfcontents
;
6333 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']==1 or listcatfiles
['ffilelist'][reallcfi
]['ftype']==2):
6334 getflinkpath
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6335 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6336 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6337 fheadersize
= format(int(flinkinfo
['fheadersize']), 'x').lower();
6338 fsize
= format(int(flinkinfo
['fsize']), 'x').lower();
6339 flinkname
= flinkinfo
['flinkname'];
6340 fatime
= format(int(flinkinfo
['fatime']), 'x').lower();
6341 fmtime
= format(int(flinkinfo
['fmtime']), 'x').lower();
6342 fctime
= format(int(flinkinfo
['fctime']), 'x').lower();
6343 fbtime
= format(int(flinkinfo
['fbtime']), 'x').lower();
6344 fmode
= format(int(flinkinfo
['fmode']), 'x').lower();
6345 fchmode
= format(int(flinkinfo
['fchmode']), 'x').lower();
6346 fuid
= format(int(flinkinfo
['fuid']), 'x').lower();
6347 funame
= flinkinfo
['funame'];
6348 fgid
= format(int(flinkinfo
['fgid']), 'x').lower();
6349 fgname
= flinkinfo
['fgname'];
6350 finode
= format(int(flinkinfo
['finode']), 'x').lower();
6351 flinkcount
= format(int(flinkinfo
['flinkcount']), 'x').lower();
6352 fwinattributes
= format(int(flinkinfo
['fwinattributes']), 'x').lower();
6353 fcompression
= flinkinfo
['fcompression'];
6354 fcsize
= format(int(flinkinfo
['fcsize']), 'x').lower();
6355 fdev_minor
= format(int(flinkinfo
['fminor']), 'x').lower();
6356 fdev_major
= format(int(flinkinfo
['fmajor']), 'x').lower();
6357 frdev_minor
= format(int(flinkinfo
['frminor']), 'x').lower();
6358 frdev_major
= format(int(flinkinfo
['frmajor']), 'x').lower();
6359 fseeknextfile
= flinkinfo
['fseeknextfile'];
6360 if(len(flinkinfo
['fextralist'])>flinkinfo
['fextrafields'] and len(flinkinfo
['fextralist'])>0):
6361 flinkinfo
['fextrafields'] = len(flinkinfo
['fextralist']);
6362 if(len(extradata
)<0):
6363 extradata
= flinkinfo
['fextralist'];
6364 fcontents
= flinkinfo
['fcontents'];
6365 ftypehex
= format(flinkinfo
['ftype'], 'x').lower();
6367 ftypehex
= format(listcatfiles
['ffilelist'][reallcfi
]['ftype'], 'x').lower();
6368 fcurfid
= format(curfid
, 'x').lower();
6369 if(not followlink
and finode
!=0):
6370 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']!=1):
6371 fcurinode
= format(int(curinode
), 'x').lower();
6372 inodetofile
.update({curinode
: fname
});
6373 filetoinode
.update({fname
: curinode
});
6374 curinode
= curinode
+ 1;
6376 fcurinode
= format(int(filetoinode
[flinkname
]), 'x').lower();
6378 fcurinode
= format(int(curinode
), 'x').lower();
6379 curinode
= curinode
+ 1;
6380 curfid
= curfid
+ 1;
6381 if(fcompression
=="none"):
6383 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, fseeknextfile
];
6384 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
6387 reallcfi
= reallcfi
+ 1;
6389 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
6390 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6391 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6394 os
.fsync(catfp
.fileno());
6395 except io
.UnsupportedOperation
:
6397 except AttributeError:
6399 except OSError as e
:
6403 if(hasattr(sys
.stdout
, "buffer")):
6404 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
6406 shutil
.copyfileobj(catfp
, sys
.stdout
);
6407 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6408 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6410 upload_file_to_internet_file(catfp
, outfile
);
6418 create_alias_function("RePack", __file_format_name__
, "", RePackArchiveFile
);
6420 def RePackArchiveFileFromString(catstr
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6421 formatspecs
= FormatSpecsListToDict(formatspecs
);
6422 catfp
= BytesIO(catstr
);
6423 listcatfiles
= RePackArchiveFile(catfp
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6424 return listcatfiles
;
6426 create_alias_function("RePack", __file_format_name__
, "FromString", RePackArchiveFileFromString
);
6428 def PackArchiveFileFromListDir(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6429 formatspecs
= FormatSpecsListToDict(formatspecs
);
6430 outarray
= BytesIO();
6431 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6432 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6433 return listcatfiles
;
6435 create_alias_function("Pack", __file_format_name__
, "FromListDir", PackArchiveFileFromListDir
);
6437 def UnPackArchiveFile(infile
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, preservepermissions
=True, preservetime
=True, verbose
=False, returnfp
=False):
6438 formatspecs
= FormatSpecsListToDict(formatspecs
);
6439 if(outdir
is not None):
6440 outdir
= RemoveWindowsPath(outdir
);
6442 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6443 if(isinstance(infile
, dict)):
6444 listcatfiles
= infile
;
6446 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6447 infile
= RemoveWindowsPath(infile
);
6448 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6449 if(not listcatfiles
):
6451 lenlist
= len(listcatfiles
['ffilelist']);
6452 fnumfiles
= int(listcatfiles
['fnumfiles']);
6454 lcfx
= int(listcatfiles
['fnumfiles']);
6455 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6456 lcfx
= int(lenlist
);
6458 lcfx
= int(listcatfiles
['fnumfiles']);
6464 userinfo
= pwd
.getpwuid(listcatfiles
['ffilelist'][lcfi
]['fuid']);
6465 funame
= userinfo
.pw_name
;
6474 groupinfo
= grp
.getgrgid(listcatfiles
['ffilelist'][lcfi
]['fgid']);
6475 fgname
= groupinfo
.gr_name
;
6481 VerbosePrintOut(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6482 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6483 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6484 listcatfiles
['ffilelist'][lcfi
]['fcontents'].seek(0, 0);
6485 shutil
.copyfileobj(listcatfiles
['ffilelist'][lcfi
]['fcontents'], fpc
);
6488 os
.fsync(fpc
.fileno());
6489 except io
.UnsupportedOperation
:
6491 except AttributeError:
6493 except OSError as e
:
6495 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6496 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6497 if(preservepermissions
):
6498 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6500 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6501 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6503 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6504 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6505 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6510 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6511 funame
= userinfo
.pw_name
;
6520 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6521 fgname
= groupinfo
.gr_name
;
6526 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6527 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6528 flinkinfo
['fcontents'].seek(0, 0);
6529 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6532 os
.fsync(fpc
.fileno());
6533 except io
.UnsupportedOperation
:
6535 except AttributeError:
6537 except OSError as e
:
6539 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6540 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6541 if(preservepermissions
):
6542 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6544 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6545 if(flinkinfo
['ftype']==1):
6546 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6547 if(flinkinfo
['ftype']==2):
6548 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6549 if(flinkinfo
['ftype']==5):
6550 if(preservepermissions
):
6551 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6553 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6554 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6555 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6556 if(preservepermissions
):
6557 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6559 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6560 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6561 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6563 os
.link(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6564 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6566 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6567 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6568 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6573 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6574 funame
= userinfo
.pw_name
;
6583 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6584 fgname
= groupinfo
.gr_name
;
6589 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6590 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6591 flinkinfo
['fcontents'].seek(0, 0);
6592 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6595 os
.fsync(fpc
.fileno());
6596 except io
.UnsupportedOperation
:
6598 except AttributeError:
6600 except OSError as e
:
6602 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6603 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6604 if(preservepermissions
):
6605 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6607 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6608 if(flinkinfo
['ftype']==1):
6609 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6610 if(flinkinfo
['ftype']==2):
6611 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6612 if(flinkinfo
['ftype']==5):
6613 if(preservepermissions
):
6614 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6616 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6617 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6618 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6619 if(preservepermissions
):
6620 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6622 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6623 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6624 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6626 os
.symlink(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6627 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6628 if(preservepermissions
):
6629 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6631 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6632 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6633 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6634 if(preservepermissions
):
6635 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6637 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6638 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6 and hasattr(os
, "mkfifo")):
6639 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6642 return listcatfiles
['ffilelist']['catfp'];
6646 create_alias_function("UnPack", __file_format_name__
, "", UnPackArchiveFile
);
6648 if(hasattr(shutil
, "register_unpack_format")):
6649 def UnPackArchiveFileFunc(archive_name
, extract_dir
=None, **kwargs
):
6650 return UnPackArchiveFile(archive_name
, extract_dir
, False, 0, 0, False, __file_format_dict__
['format_delimiter'], False, False);
6651 create_alias_function("UnPack", __file_format_name__
, "Func", UnPackArchiveFileFunc
);
6653 def UnPackArchiveFileString(catstr
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6654 formatspecs
= FormatSpecsListToDict(formatspecs
);
6655 catfp
= BytesIO(catstr
);
6656 listcatfiles
= UnPackArchiveFile(catfp
, outdir
, followlink
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6657 return listcatfiles
;
6659 create_alias_function("UnPack", __file_format_name__
, "String", UnPackArchiveFileString
);
6661 def ArchiveFileListFiles(infile
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6662 formatspecs
= FormatSpecsListToDict(formatspecs
);
6663 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6664 if(isinstance(infile
, dict)):
6665 listcatfiles
= infile
;
6667 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6668 infile
= RemoveWindowsPath(infile
);
6669 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, True, False, skipchecksum
, formatspecs
, returnfp
);
6670 if(not listcatfiles
):
6672 lenlist
= len(listcatfiles
['ffilelist']);
6673 fnumfiles
= int(listcatfiles
['fnumfiles']);
6675 lcfx
= int(listcatfiles
['fnumfiles']);
6676 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6677 lcfx
= int(lenlist
);
6679 lcfx
= int(listcatfiles
['fnumfiles']);
6682 returnval
.update({lcfi
: listcatfiles
['ffilelist'][lcfi
]['fname']});
6684 VerbosePrintOut(listcatfiles
['ffilelist'][lcfi
]['fname']);
6686 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6687 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'];
6688 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6689 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " link to " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6690 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6691 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " -> " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6692 fuprint
= listcatfiles
['ffilelist'][lcfi
]['funame'];
6693 if(len(fuprint
)<=0):
6694 fuprint
= listcatfiles
['ffilelist'][lcfi
]['fuid'];
6695 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgname'];
6696 if(len(fgprint
)<=0):
6697 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgid'];
6698 VerbosePrintOut(PrintPermissionString(listcatfiles
['ffilelist'][lcfi
]['fmode'], listcatfiles
['ffilelist'][lcfi
]['ftype']) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(listcatfiles
['ffilelist'][lcfi
]['fsize']).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(listcatfiles
['ffilelist'][lcfi
]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6701 return listcatfiles
['catfp'];
6705 create_alias_function("", __file_format_name__
, "ListFiles", ArchiveFileListFiles
);
6707 def ArchiveFileStringListFiles(catstr
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6708 formatspecs
= FormatSpecsListToDict(formatspecs
);
6709 catfp
= BytesIO(catstr
);
6710 listcatfiles
= ArchiveFileListFiles(catstr
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6711 return listcatfiles
;
6713 create_alias_function("", __file_format_name__
, "StringListFiles", ArchiveFileStringListFiles
);
6715 def TarFileListFiles(infile
, verbose
=False, returnfp
=False):
6716 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6719 if(hasattr(sys
.stdin
, "buffer")):
6720 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6722 shutil
.copyfileobj(sys
.stdin
, infile
);
6727 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6728 infile
= download_file_from_internet_file(infile
);
6733 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6735 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
6737 if(not tarfile
.TarFileCheck(infile
)):
6739 except AttributeError:
6740 if(not TarFileCheck(infile
)):
6745 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
6746 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
6748 tarfp
= tarfile
.open(infile
, "r");
6749 except FileNotFoundError
:
6753 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
6754 returnval
.update({lcfi
: member
.name
});
6755 fpremode
= member
.mode
;
6756 ffullmode
= member
.mode
;
6760 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6762 elif(member
.isdev()):
6763 ffullmode
= member
.mode
;
6765 elif(member
.islnk()):
6766 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6768 elif(member
.issym()):
6769 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
6771 elif(member
.ischr()):
6772 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
6774 elif(member
.isblk()):
6775 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
6777 elif(member
.isdir()):
6778 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
6780 elif(member
.isfifo()):
6781 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
6783 elif(member
.issparse()):
6784 ffullmode
= member
.mode
;
6787 VerbosePrintOut(member
.name
);
6789 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6790 printfname
= member
.name
;
6792 printfname
= member
.name
+ " link to " + member
.linkname
;
6793 elif(member
.issym()):
6794 printfname
= member
.name
+ " -> " + member
.linkname
;
6795 fuprint
= member
.uname
;
6796 if(len(fuprint
)<=0):
6797 fuprint
= member
.uid
;
6798 fgprint
= member
.gname
;
6799 if(len(fgprint
)<=0):
6800 fgprint
= member
.gid
;
6801 VerbosePrintOut(PrintPermissionString(ffullmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(member
.mtime
).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6804 return listcatfiles
['catfp'];
6808 def ZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6809 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6812 if(hasattr(sys
.stdin
, "buffer")):
6813 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6815 shutil
.copyfileobj(sys
.stdin
, infile
);
6820 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6821 infile
= download_file_from_internet_file(infile
);
6826 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6830 if(not zipfile
.is_zipfile(infile
)):
6833 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
6834 except FileNotFoundError
:
6838 ziptest
= zipfp
.testzip();
6840 VerbosePrintOut("Bad file found!");
6841 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
6842 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
6843 fwinattributes
= int(zipinfo
.external_attr
);
6844 if(not member
.is_dir()):
6845 fmode
= int(stat
.S_IFREG
+ 438);
6846 fchmode
= int(stat
.S_IMODE(fmode
));
6847 ftypemod
= int(stat
.S_IFMT(fmode
));
6848 elif(member
.is_dir()):
6849 fmode
= int(stat
.S_IFDIR
+ 511);
6850 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6851 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6852 elif(zipinfo
.create_system
==3):
6853 fwinattributes
=int(0);
6854 fmode
= int(zipinfo
.external_attr
);
6855 fchmode
= int(stat
.S_IMODE(fmode
));
6856 ftypemod
= int(stat
.S_IFMT(fmode
));
6858 fwinattributes
= int(0);
6859 if(not member
.is_dir()):
6860 fmode
= int(stat
.S_IFREG
+ 438);
6861 fchmode
= int(stat
.S_IMODE(fmode
));
6862 ftypemod
= int(stat
.S_IFMT(fmode
));
6863 elif(member
.is_dir()):
6864 fmode
= int(stat
.S_IFDIR
+ 511);
6865 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6866 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6867 returnval
.update({lcfi
: member
.filename
});
6869 VerbosePrintOut(member
.filename
);
6871 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6873 for fmodval
in str(oct(fmode
))[-3:]:
6874 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6875 if(not member
.is_dir()):
6877 permissionstr
= "-" + permissionstr
;
6878 elif(member
.is_dir()):
6880 permissionstr
= "d" + permissionstr
;
6881 printfname
= member
.filename
;
6883 fuid
= int(os
.getuid());
6884 except AttributeError:
6889 fgid
= int(os
.getgid());
6890 except AttributeError:
6897 userinfo
= pwd
.getpwuid(os
.getuid());
6898 funame
= userinfo
.pw_name
;
6901 except AttributeError:
6909 groupinfo
= grp
.getgrgid(os
.getgid());
6910 fgname
= groupinfo
.gr_name
;
6913 except AttributeError:
6918 if(len(fuprint
)<=0):
6919 fuprint
= str(fuid
);
6921 if(len(fgprint
)<=0):
6922 fgprint
= str(fgid
);
6923 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(int(time
.mktime(member
.date_time
+ (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6926 return listcatfiles
['catfp'];
6930 if(not rarfile_support
):
6931 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6932 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6933 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6936 if(rarfile_support
):
6937 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6938 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6939 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6941 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
6945 rarfp
= rarfile
.RarFile(infile
, "r");
6946 rartest
= rarfp
.testrar();
6948 VerbosePrintOut("Bad file found!");
6949 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
6952 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
6955 member
.external_attr
6957 except AttributeError:
6959 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
6962 member
.external_attr
6964 except AttributeError:
6969 if(is_unix
and member
.external_attr
!=0):
6970 fpremode
= int(member
.external_attr
);
6971 elif(member
.is_file()):
6972 fpremode
= int(stat
.S_IFREG
+ 438);
6973 elif(member
.is_symlink()):
6974 fpremode
= int(stat
.S_IFLNK
+ 438);
6975 elif(member
.is_dir()):
6976 fpremode
= int(stat
.S_IFDIR
+ 511);
6977 if(is_windows
and member
.external_attr
!=0):
6978 fwinattributes
= int(member
.external_attr
);
6980 fwinattributes
= int(0);
6981 if(is_unix
and member
.external_attr
!=0):
6982 fmode
= int(member
.external_attr
);
6983 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
6984 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
6985 elif(member
.is_file()):
6986 fmode
= int(stat
.S_IFREG
+ 438);
6987 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
6988 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
6989 elif(member
.is_symlink()):
6990 fmode
= int(stat
.S_IFLNK
+ 438);
6991 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
6992 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
6993 elif(member
.is_dir()):
6994 fmode
= int(stat
.S_IFDIR
+ 511);
6995 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6996 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6997 returnval
.update({lcfi
: member
.filename
});
6999 VerbosePrintOut(member
.filename
);
7001 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7003 for fmodval
in str(oct(fmode
))[-3:]:
7004 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7005 if(member
.is_file()):
7007 permissionstr
= "-" + permissionstr
;
7008 printfname
= member
.filename
;
7009 elif(member
.is_symlink()):
7011 permissionstr
= "l" + permissionstr
;
7012 printfname
= member
.name
+ " -> " + member
.read().decode("UTF-8");
7013 elif(member
.is_dir()):
7015 permissionstr
= "d" + permissionstr
;
7016 printfname
= member
.filename
;
7018 fuid
= int(os
.getuid());
7019 except AttributeError:
7024 fgid
= int(os
.getgid());
7025 except AttributeError:
7032 userinfo
= pwd
.getpwuid(os
.getuid());
7033 funame
= userinfo
.pw_name
;
7036 except AttributeError:
7044 groupinfo
= grp
.getgrgid(os
.getgid());
7045 fgname
= groupinfo
.gr_name
;
7048 except AttributeError:
7053 if(len(fuprint
)<=0):
7054 fuprint
= str(fuid
);
7056 if(len(fgprint
)<=0):
7057 fgprint
= str(fgid
);
7058 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + member
.mtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7061 return listcatfiles
['catfp'];
7065 if(not py7zr_support
):
7066 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7067 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7068 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7072 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7073 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7074 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7078 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
7079 file_content
= szpfp
.readall();
7080 #sztest = szpfp.testzip();
7081 sztestalt
= szpfp
.test();
7083 VerbosePrintOut("Bad file found!");
7084 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
7085 if(re
.findall(r
"^[.|/]", member
.filename
)):
7086 fname
= member
.filename
;
7088 fname
= "./"+member
.filename
;
7089 if(not member
.is_directory
):
7090 fpremode
= int(stat
.S_IFREG
+ 438);
7091 elif(member
.is_directory
):
7092 fpremode
= int(stat
.S_IFDIR
+ 511);
7093 fwinattributes
= int(0);
7094 if(member
.is_directory
):
7095 fmode
= int(stat
.S_IFDIR
+ 511);
7096 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7097 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7099 fmode
= int(stat
.S_IFLNK
+ 438);
7100 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7101 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7102 returnval
.update({lcfi
: member
.filename
});
7104 VerbosePrintOut(member
.filename
);
7106 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7108 for fmodval
in str(oct(fmode
))[-3:]:
7109 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7111 if(not member
.is_directory
):
7113 permissionstr
= "-" + permissionstr
;
7114 printfname
= member
.filename
;
7115 elif(member
.is_directory
):
7117 permissionstr
= "d" + permissionstr
;
7118 printfname
= member
.filename
;
7120 fsize
= len(file_content
[member
.filename
].read());
7121 file_content
[member
.filename
].close();
7123 fuid
= int(os
.getuid());
7124 except AttributeError:
7129 fgid
= int(os
.getgid());
7130 except AttributeError:
7137 userinfo
= pwd
.getpwuid(os
.getuid());
7138 funame
= userinfo
.pw_name
;
7141 except AttributeError:
7149 groupinfo
= grp
.getgrgid(os
.getgid());
7150 fgname
= groupinfo
.gr_name
;
7153 except AttributeError:
7158 if(len(fuprint
)<=0):
7159 fuprint
= str(fuid
);
7161 if(len(fgprint
)<=0):
7162 fgprint
= str(fgid
);
7163 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(fsize
).rjust(15) + " " + member
.creationtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7166 return listcatfiles
['catfp'];
7170 def InFileListFiles(infile
, verbose
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
7171 formatspecs
= FormatSpecsListToDict(formatspecs
);
7172 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7173 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
7174 if(checkcompressfile
=="tarfile"):
7175 return TarFileListFiles(infile
, verbose
, returnfp
);
7176 elif(checkcompressfile
=="zipfile"):
7177 return ZipFileListFiles(infile
, verbose
, returnfp
);
7178 elif(checkcompressfile
=="catfile"):
7179 return ArchiveFileListFiles(infile
, 0, 0, False, formatspecs
, verbose
, returnfp
);
7180 elif(rarfile_support
and checkcompressfile
=="rarfile"):
7181 return RarFileListFiles(infile
, verbose
, returnfp
);
7182 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
7183 return SevenZipFileListFiles(infile
, verbose
, returnfp
);
7188 def ListDirListFiles(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7189 formatspecs
= FormatSpecsListToDict(formatspecs
);
7190 outarray
= BytesIO();
7191 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, False, True);
7192 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7193 return listcatfiles
;
7195 def ListDirListFilesAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7196 formatspecs
= FormatSpecsListToDict(formatspecs
);
7197 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, formatspecs
, verbose
);
7198 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7199 return listcatfiles
;
7201 def PackArchiveFileFromListDirAlt(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7202 formatspecs
= FormatSpecsListToDict(formatspecs
);
7203 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, False, checksumtype
, extradata
, formatspecs
, False);
7204 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
7205 return listcatfiles
;
7207 create_alias_function("Pack", __file_format_name__
, "FromListDirAlt", PackArchiveFileFromListDirAlt
);
7209 def PackArchiveFileFromTarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7210 formatspecs
= FormatSpecsListToDict(formatspecs
);
7211 outarray
= TarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7212 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7213 return listcatfiles
;
7215 create_alias_function("Pack", __file_format_name__
, "FromTarFileAlt", PackArchiveFileFromTarFileAlt
);
7217 def PackArchiveFileFromZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7218 formatspecs
= FormatSpecsListToDict(formatspecs
);
7219 outarray
= ZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7220 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7221 return listcatfiles
;
7223 create_alias_function("Pack", __file_format_name__
, "FromZipFileAlt", PackArchiveFileFromZipFileAlt
);
7225 if(not rarfile_support
):
7226 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7229 if(rarfile_support
):
7230 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7231 formatspecs
= FormatSpecsListToDict(formatspecs
);
7232 outarray
= RarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7233 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7234 return listcatfiles
;
7236 create_alias_function("Pack", __file_format_name__
, "FromRarFileAlt", PackArchiveFileFromRarFileAlt
);
7238 if(not py7zr_support
):
7239 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7243 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7244 formatspecs
= FormatSpecsListToDict(formatspecs
);
7245 outarray
= SevenZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7246 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7247 return listcatfiles
;
7249 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt
);
7251 def download_file_from_ftp_file(url
):
7252 urlparts
= urlparse(url
);
7253 file_name
= os
.path
.basename(urlparts
.path
);
7254 file_dir
= os
.path
.dirname(urlparts
.path
);
7255 if(urlparts
.username
is not None):
7256 ftp_username
= urlparts
.username
;
7258 ftp_username
= "anonymous";
7259 if(urlparts
.password
is not None):
7260 ftp_password
= urlparts
.password
;
7261 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7262 ftp_password
= "anonymous";
7265 if(urlparts
.scheme
=="ftp"):
7267 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7271 if(urlparts
.scheme
=="sftp"):
7273 return download_file_from_pysftp_file(url
);
7275 return download_file_from_sftp_file(url
);
7276 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7277 return download_file_from_http_file(url
);
7278 ftp_port
= urlparts
.port
;
7279 if(urlparts
.port
is None):
7282 ftp
.connect(urlparts
.hostname
, ftp_port
);
7283 except socket
.gaierror
:
7284 log
.info("Error With URL "+url
);
7286 except socket
.timeout
:
7287 log
.info("Error With URL "+url
);
7289 ftp
.login(urlparts
.username
, urlparts
.password
);
7290 if(urlparts
.scheme
=="ftps"):
7292 ftpfile
= BytesIO();
7293 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
7294 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7299 def download_file_from_ftp_string(url
):
7300 ftpfile
= download_file_from_ftp_file(url
);
7301 return ftpfile
.read();
7303 def upload_file_to_ftp_file(ftpfile
, url
):
7304 urlparts
= urlparse(url
);
7305 file_name
= os
.path
.basename(urlparts
.path
);
7306 file_dir
= os
.path
.dirname(urlparts
.path
);
7307 if(urlparts
.username
is not None):
7308 ftp_username
= urlparts
.username
;
7310 ftp_username
= "anonymous";
7311 if(urlparts
.password
is not None):
7312 ftp_password
= urlparts
.password
;
7313 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7314 ftp_password
= "anonymous";
7317 if(urlparts
.scheme
=="ftp"):
7319 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7323 if(urlparts
.scheme
=="sftp"):
7325 return upload_file_to_pysftp_file(url
);
7327 return upload_file_to_sftp_file(url
);
7328 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7330 ftp_port
= urlparts
.port
;
7331 if(urlparts
.port
is None):
7334 ftp
.connect(urlparts
.hostname
, ftp_port
);
7335 except socket
.gaierror
:
7336 log
.info("Error With URL "+url
);
7338 except socket
.timeout
:
7339 log
.info("Error With URL "+url
);
7341 ftp
.login(urlparts
.username
, urlparts
.password
);
7342 if(urlparts
.scheme
=="ftps"):
7344 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
7349 def upload_file_to_ftp_string(ftpstring
, url
):
7350 ftpfileo
= BytesIO(ftpstring
);
7351 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
7355 def download_file_from_http_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7356 # Parse the URL to extract username and password if present
7357 urlparts
= urlparse(url
);
7358 username
= urlparts
.username
;
7359 password
= urlparts
.password
;
7360 # Rebuild the URL without the username and password
7361 netloc
= urlparts
.hostname
;
7362 if(urlparts
.scheme
=="sftp"):
7364 return download_file_from_pysftp_file(url
);
7366 return download_file_from_sftp_file(url
);
7367 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7368 return download_file_from_ftp_file(url
);
7370 netloc
+= ':' + str(urlparts
.port
);
7371 rebuilt_url
= urlunparse((urlparts
.scheme
, netloc
, urlparts
.path
, urlparts
.params
, urlparts
.query
, urlparts
.fragment
));
7372 # Create a temporary file object
7373 httpfile
= BytesIO();
7375 # Use the requests library if available
7376 if username
and password
:
7377 response
= requests
.get(rebuilt_url
, headers
=headers
, auth
=(username
, password
), stream
=True);
7379 response
= requests
.get(rebuilt_url
, headers
=headers
, stream
=True);
7380 response
.raw
.decode_content
= True
7381 shutil
.copyfileobj(response
.raw
, httpfile
);
7383 # Build a Request object for urllib
7384 request
= Request(rebuilt_url
, headers
=headers
);
7385 # Create an opener object for handling URLs
7386 if username
and password
:
7387 # Create a password manager
7388 password_mgr
= HTTPPasswordMgrWithDefaultRealm();
7389 # Add the username and password
7390 password_mgr
.add_password(None, rebuilt_url
, username
, password
);
7391 # Create an authentication handler using the password manager
7392 auth_handler
= HTTPBasicAuthHandler(password_mgr
);
7393 # Build the opener with the authentication handler
7394 opener
= build_opener(auth_handler
);
7396 opener
= build_opener();
7397 with opener
.open(request
) as response
:
7398 shutil
.copyfileobj(response
, httpfile
);
7399 # Reset file pointer to the start
7400 httpfile
.seek(0, 0);
7401 # Return the temporary file object
7404 def download_file_from_http_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7405 httpfile
= download_file_from_http_file(url
, headers
);
7406 return ftpfile
.read();
7409 def download_file_from_sftp_file(url
):
7410 urlparts
= urlparse(url
);
7411 file_name
= os
.path
.basename(urlparts
.path
);
7412 file_dir
= os
.path
.dirname(urlparts
.path
);
7413 sftp_port
= urlparts
.port
;
7414 if(urlparts
.port
is None):
7417 sftp_port
= urlparts
.port
;
7418 if(urlparts
.username
is not None):
7419 sftp_username
= urlparts
.username
;
7421 sftp_username
= "anonymous";
7422 if(urlparts
.password
is not None):
7423 sftp_password
= urlparts
.password
;
7424 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7425 sftp_password
= "anonymous";
7428 if(urlparts
.scheme
=="ftp"):
7429 return download_file_from_ftp_file(url
);
7430 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7431 return download_file_from_http_file(url
);
7432 if(urlparts
.scheme
!="sftp"):
7434 ssh
= paramiko
.SSHClient();
7435 ssh
.load_system_host_keys();
7436 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7438 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7439 except paramiko
.ssh_exception
.SSHException
:
7441 except socket
.gaierror
:
7442 log
.info("Error With URL "+url
);
7444 except socket
.timeout
:
7445 log
.info("Error With URL "+url
);
7447 sftp
= ssh
.open_sftp();
7448 sftpfile
= BytesIO();
7449 sftp
.getfo(urlparts
.path
, sftpfile
);
7452 sftpfile
.seek(0, 0);
7455 def download_file_from_sftp_file(url
):
7459 def download_file_from_sftp_string(url
):
7460 sftpfile
= download_file_from_sftp_file(url
);
7461 return sftpfile
.read();
7463 def download_file_from_ftp_string(url
):
7467 def upload_file_to_sftp_file(sftpfile
, url
):
7468 urlparts
= urlparse(url
);
7469 file_name
= os
.path
.basename(urlparts
.path
);
7470 file_dir
= os
.path
.dirname(urlparts
.path
);
7471 sftp_port
= urlparts
.port
;
7472 if(urlparts
.port
is None):
7475 sftp_port
= urlparts
.port
;
7476 if(urlparts
.username
is not None):
7477 sftp_username
= urlparts
.username
;
7479 sftp_username
= "anonymous";
7480 if(urlparts
.password
is not None):
7481 sftp_password
= urlparts
.password
;
7482 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7483 sftp_password
= "anonymous";
7486 if(urlparts
.scheme
=="ftp"):
7487 return upload_file_to_ftp_file(url
);
7488 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7490 if(urlparts
.scheme
!="sftp"):
7492 ssh
= paramiko
.SSHClient();
7493 ssh
.load_system_host_keys();
7494 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7496 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7497 except paramiko
.ssh_exception
.SSHException
:
7499 except socket
.gaierror
:
7500 log
.info("Error With URL "+url
);
7502 except socket
.timeout
:
7503 log
.info("Error With URL "+url
);
7505 sftp
= ssh
.open_sftp();
7506 sftp
.putfo(sftpfile
, urlparts
.path
);
7509 sftpfile
.seek(0, 0);
7512 def upload_file_to_sftp_file(sftpfile
, url
):
7516 def upload_file_to_sftp_string(sftpstring
, url
):
7517 sftpfileo
= BytesIO(sftpstring
);
7518 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
7522 def upload_file_to_sftp_string(url
):
7526 def download_file_from_pysftp_file(url
):
7527 urlparts
= urlparse(url
);
7528 file_name
= os
.path
.basename(urlparts
.path
);
7529 file_dir
= os
.path
.dirname(urlparts
.path
);
7530 sftp_port
= urlparts
.port
;
7531 if(urlparts
.port
is None):
7534 sftp_port
= urlparts
.port
;
7535 if(urlparts
.username
is not None):
7536 sftp_username
= urlparts
.username
;
7538 sftp_username
= "anonymous";
7539 if(urlparts
.password
is not None):
7540 sftp_password
= urlparts
.password
;
7541 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7542 sftp_password
= "anonymous";
7545 if(urlparts
.scheme
=="ftp"):
7546 return download_file_from_ftp_file(url
);
7547 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7548 return download_file_from_http_file(url
);
7549 if(urlparts
.scheme
!="sftp"):
7552 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7553 except paramiko
.ssh_exception
.SSHException
:
7555 except socket
.gaierror
:
7556 log
.info("Error With URL "+url
);
7558 except socket
.timeout
:
7559 log
.info("Error With URL "+url
);
7561 sftp
= ssh
.open_sftp();
7562 sftpfile
= BytesIO();
7563 sftp
.getfo(urlparts
.path
, sftpfile
);
7566 sftpfile
.seek(0, 0);
7569 def download_file_from_pysftp_file(url
):
7573 def download_file_from_pysftp_string(url
):
7574 sftpfile
= download_file_from_pysftp_file(url
);
7575 return sftpfile
.read();
7577 def download_file_from_ftp_string(url
):
7581 def upload_file_to_pysftp_file(sftpfile
, url
):
7582 urlparts
= urlparse(url
);
7583 file_name
= os
.path
.basename(urlparts
.path
);
7584 file_dir
= os
.path
.dirname(urlparts
.path
);
7585 sftp_port
= urlparts
.port
;
7586 if(urlparts
.port
is None):
7589 sftp_port
= urlparts
.port
;
7590 if(urlparts
.username
is not None):
7591 sftp_username
= urlparts
.username
;
7593 sftp_username
= "anonymous";
7594 if(urlparts
.password
is not None):
7595 sftp_password
= urlparts
.password
;
7596 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7597 sftp_password
= "anonymous";
7600 if(urlparts
.scheme
=="ftp"):
7601 return upload_file_to_ftp_file(url
);
7602 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7604 if(urlparts
.scheme
!="sftp"):
7607 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7608 except paramiko
.ssh_exception
.SSHException
:
7610 except socket
.gaierror
:
7611 log
.info("Error With URL "+url
);
7613 except socket
.timeout
:
7614 log
.info("Error With URL "+url
);
7616 sftp
= ssh
.open_sftp();
7617 sftp
.putfo(sftpfile
, urlparts
.path
);
7620 sftpfile
.seek(0, 0);
7623 def upload_file_to_pysftp_file(sftpfile
, url
):
7627 def upload_file_to_pysftp_string(sftpstring
, url
):
7628 sftpfileo
= BytesIO(sftpstring
);
7629 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
7633 def upload_file_to_pysftp_string(url
):
7636 def download_file_from_internet_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7637 urlparts
= urlparse(url
);
7638 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7639 return download_file_from_http_file(url
, headers
);
7640 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7641 return download_file_from_ftp_file(url
);
7642 elif(urlparts
.scheme
=="sftp"):
7643 if(__use_pysftp__
and havepysftp
):
7644 return download_file_from_pysftp_file(url
);
7646 return download_file_from_sftp_file(url
);
7651 def download_file_from_internet_uncompress_file(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7652 formatspecs
= FormatSpecsListToDict(formatspecs
);
7653 fp
= download_file_from_internet_file(url
);
7654 fp
= UncompressArchiveFile(fp
, formatspecs
);
7660 def download_file_from_internet_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7661 urlparts
= urlparse(url
);
7662 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7663 return download_file_from_http_string(url
, headers
);
7664 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7665 return download_file_from_ftp_string(url
);
7666 elif(urlparts
.scheme
=="sftp"):
7667 if(__use_pysftp__
and havepysftp
):
7668 return download_file_from_pysftp_string(url
);
7670 return download_file_from_sftp_string(url
);
7675 def download_file_from_internet_uncompress_string(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7676 formatspecs
= FormatSpecsListToDict(formatspecs
);
7677 fp
= download_file_from_internet_string(url
);
7678 fp
= UncompressArchiveFile(fp
, formatspecs
);
7684 def upload_file_to_internet_file(ifp
, url
):
7685 urlparts
= urlparse(url
);
7686 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7688 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7689 return upload_file_to_ftp_file(ifp
, url
);
7690 elif(urlparts
.scheme
=="sftp"):
7691 if(__use_pysftp__
and havepysftp
):
7692 return upload_file_to_pysftp_file(ifp
, url
);
7694 return upload_file_to_sftp_file(ifp
, url
);
7699 def upload_file_to_internet_compress_file(ifp
, url
, formatspecs
=__file_format_dict__
):
7700 formatspecs
= FormatSpecsListToDict(formatspecs
);
7701 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
7705 upload_file_to_internet_file(catfp
, outfile
);
7708 def upload_file_to_internet_string(ifp
, url
):
7709 urlparts
= urlparse(url
);
7710 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7712 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7713 return upload_file_to_ftp_string(ifp
, url
);
7714 elif(urlparts
.scheme
=="sftp"):
7715 if(__use_pysftp__
and havepysftp
):
7716 return upload_file_to_pysftp_string(ifp
, url
);
7718 return upload_file_to_sftp_string(ifp
, url
);
7723 def upload_file_to_internet_compress_string(ifp
, url
, formatspecs
=__file_format_dict__
):
7724 formatspecs
= FormatSpecsListToDict(formatspecs
);
7725 catfp
= CompressArchiveFile(BytesIO(ifp
), compression
, compressionlevel
, formatspecs
);
7729 upload_file_to_internet_file(catfp
, outfile
);
7733 if(hasattr(shutil
, "register_archive_format")):
7734 # Register the packing format
7735 shutil
.register_archive_format(__file_format_name__
, PackArchiveFileFunc
, description
='Pack concatenated files');
7736 except shutil
.RegistryError
:
7740 if(hasattr(shutil
, "register_unpack_format")):
7741 # Register the unpacking format
7742 shutil
.register_unpack_format(__file_format_name__
, archivefile_extensions
, UnPackArchiveFileFunc
, description
='UnPack concatenated files');
7743 except shutil
.RegistryError
: