native_client_sdk/src/tools/oshelpers.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import fnmatch
   7 import glob
   8 import optparse
   9 import os
  10 import posixpath
  11 import shutil
  12 import sys
  13 import time
  14 import zipfile
  15
  16
  17 def IncludeFiles(filters, files):
  18   """Filter files based on inclusion lists
  19
  20   Return a list of files which match and of the Unix shell-style wildcards
  21   provided, or return all the files if no filter is provided."""
  22   if not filters:
  23     return files
  24   match = set()
  25   for filter in filters:
  26     match |= set(fnmatch.filter(files, filter))
  27   return [name for name in files if name in match]
  28
  29
  30 def ExcludeFiles(filters, files):
  31   """Filter files based on exclusions lists
  32
  33   Return a list of files which do not match any of the Unix shell-style
  34   wildcards provided, or return all the files if no filter is provided."""
  35   if not filters:
  36     return files
  37   match = set()
  38   for filter in filters:
  39     excludes = set(fnmatch.filter(files, filter))
  40     match |= excludes
  41   return [name for name in files if name not in match]
  42
  43
  44 def CopyPath(options, src, dst):
  45   """CopyPath from src to dst
  46
  47   Copy a fully specified src to a fully specified dst.  If src and dst are
  48   both files, the dst file is removed first to prevent error.  If and include
  49   or exclude list are provided, the destination is first matched against that
  50   filter."""
  51   if options.includes:
  52     if not IncludeFiles(options.includes, [src]):
  53       return
  54
  55   if options.excludes:
  56     if not ExcludeFiles(options.excludes, [src]):
  57       return
  58
  59   if options.verbose:
  60     print 'cp %s %s' % (src, dst)
  61
  62   # If the source is a single file, copy it individually
  63   if os.path.isfile(src):
  64     # We can not copy over a directory with a file.
  65     if os.path.exists(dst):
  66       if not os.path.isfile(dst):
  67         msg = "cp: cannot overwrite non-file '%s' with file." % dst
  68         raise OSError(msg)
  69       # If the destination exists as a file, remove it before copying to avoid
  70       # 'readonly' issues.
  71       os.remove(dst)
  72
  73     # Now copy to the non-existent fully qualified target
  74     shutil.copy(src, dst)
  75     return
  76
  77   # Otherwise it's a directory, ignore it unless allowed
  78   if os.path.isdir(src):
  79     if not options.recursive:
  80       print "cp: omitting directory '%s'" % src
  81       return
  82
  83     # We can not copy over a file with a directory.
  84     if os.path.exists(dst):
  85       if not os.path.isdir(dst):
  86         msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
  87         raise OSError(msg)
  88     else:
  89       # if it didn't exist, create the directory
  90       os.makedirs(dst)
  91
  92     # Now copy all members
  93     for filename in os.listdir(src):
  94       srcfile = os.path.join(src, filename)
  95       dstfile = os.path.join(dst, filename)
  96       CopyPath(options, srcfile, dstfile)
  97   return
  98
  99
 100 def Copy(args):
 101   """A Unix cp style copy.
 102
 103   Copies multiple sources to a single destination using the normal cp
 104   semantics.  In addition, it support inclusion and exclusion filters which
 105   allows the copy to skip certain types of files."""
 106   parser = optparse.OptionParser(usage='usage: cp [Options] souces... dest')
 107   parser.add_option(
 108       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 109       default=False,
 110       help='copy directories recursively.')
 111   parser.add_option(
 112       '-v', '--verbose', dest='verbose', action='store_true',
 113       default=False,
 114       help='verbose output.')
 115   parser.add_option(
 116       '--include', dest='includes', action='append', default=[],
 117       help='include files matching this expression.')
 118   parser.add_option(
 119       '--exclude', dest='excludes', action='append', default=[],
 120       help='exclude files matching this expression.')
 121   options, files = parser.parse_args(args)
 122   if len(files) < 2:
 123     parser.error('ERROR: expecting SOURCE(s) and DEST.')
 124
 125   srcs = files[:-1]
 126   dst = files[-1]
 127
 128   src_list = []
 129   for src in srcs:
 130     files = glob.glob(src)
 131     if len(files) == 0:
 132       raise OSError('cp: no such file or directory: ' + src)
 133     if files:
 134       src_list.extend(files)
 135
 136   for src in src_list:
 137     # If the destination is a directory, then append the basename of the src
 138     # to the destination.
 139     if os.path.isdir(dst):
 140       CopyPath(options, src, os.path.join(dst, os.path.basename(src)))
 141     else:
 142       CopyPath(options, src, dst)
 143
 144
 145 def Mkdir(args):
 146   """A Unix style mkdir"""
 147   parser = optparse.OptionParser(usage='usage: mkdir [Options] DIRECTORY...')
 148   parser.add_option(
 149       '-p', '--parents', dest='parents', action='store_true',
 150       default=False,
 151       help='ignore existing parents, create parents as needed.')
 152   parser.add_option(
 153       '-v', '--verbose', dest='verbose', action='store_true',
 154       default=False,
 155       help='verbose output.')
 156
 157   options, dsts = parser.parse_args(args)
 158   if len(dsts) < 1:
 159     parser.error('ERROR: expecting DIRECTORY...')
 160
 161   for dst in dsts:
 162     if options.verbose:
 163       print 'mkdir ' + dst
 164     try:
 165       os.makedirs(dst)
 166     except OSError as error:
 167       if os.path.isdir(dst):
 168         if options.parents:
 169           continue
 170         raise OSError('mkdir: Already exsists: ' + dst)
 171       else:
 172         raise OSError('mkdir: Failed to create: ' + dst)
 173   return 0
 174
 175
 176 def MovePath(options, src, dst):
 177   """MovePath from src to dst
 178
 179   Moves the src to the dst much like the Unix style mv command, except it
 180   only handles one source at a time.  Because of possible temporary failures
 181   do to locks (such as anti-virus software on Windows), the function will retry
 182   up to five times."""
 183   # if the destination is not an existing directory, then overwrite it
 184   if os.path.isdir(dst):
 185     dst = os.path.join(dst, os.path.basename(src))
 186
 187   # If the destination exists, the remove it
 188   if os.path.exists(dst):
 189     if options.force:
 190       Remove(['-vfr', dst])
 191       if os.path.exists(dst):
 192         raise OSError('mv: FAILED TO REMOVE ' + dst)
 193     else:
 194       raise OSError('mv: already exists ' + dst)
 195   for i in range(5):
 196     try:
 197       os.rename(src, dst)
 198       return
 199     except OSError as error:
 200       print 'Failed on %s with %s, retrying' % (src, error)
 201       time.sleep(5)
 202   print 'Gave up.'
 203   raise OSError('mv: ' + error)
 204
 205
 206 def Move(args):
 207   parser = optparse.OptionParser(usage='usage: mv [Options] souces... dest')
 208   parser.add_option(
 209       '-v', '--verbose', dest='verbose', action='store_true',
 210       default=False,
 211       help='verbose output.')
 212   parser.add_option(
 213       '-f', '--force', dest='force', action='store_true',
 214       default=False,
 215       help='force, do not error it files already exist.')
 216   options, files = parser.parse_args(args)
 217   if len(files) < 2:
 218     parser.error('ERROR: expecting SOURCE... and DEST.')
 219   if options.verbose:
 220     print 'mv %s %s' % (src, dst)
 221
 222   srcs = files[:-1]
 223   dst = files[-1]
 224
 225   for src in srcs:
 226     MovePath(options, src, dst)
 227   return 0
 228
 229
 230 def Remove(args):
 231   """A Unix style rm.
 232
 233   Removes the list of paths.  Because of possible temporary failures do to locks
 234   (such as anti-virus software on Windows), the function will retry up to five
 235   times."""
 236   parser = optparse.OptionParser(usage='usage: rm [Options] PATHS...')
 237   parser.add_option(
 238       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 239       default=False,
 240       help='remove directories recursively.')
 241   parser.add_option(
 242       '-v', '--verbose', dest='verbose', action='store_true',
 243       default=False,
 244       help='verbose output.')
 245   parser.add_option(
 246       '-f', '--force', dest='force', action='store_true',
 247       default=False,
 248       help='force, do not error it files does not exist.')
 249   options, files = parser.parse_args(args)
 250   if len(files) < 1:
 251     parser.error('ERROR: expecting FILE...')
 252
 253   try:
 254     for pattern in files:
 255       dst_files = glob.glob(pattern)
 256       # Ignore non existing files when using force
 257       if len(dst_files) == 0 and options.force:
 258         print "rm: Skipping " + pattern
 259         continue
 260       elif len(dst_files) == 0:
 261         raise OSError('rm: no such file or directory: ' + pattern)
 262
 263       for dst in dst_files:
 264         if options.verbose:
 265           print 'rm ' + dst
 266
 267         if os.path.isfile(dst) or os.path.islink(dst):
 268           for i in range(5):
 269             try:
 270               # Check every time, since it may have been deleted after the
 271               # previous failed attempt.
 272               if os.path.isfile(dst) or os.path.islink(dst):
 273                 os.remove(dst)
 274               break
 275             except OSError as error:
 276               if i == 5:
 277                 print 'Gave up.'
 278                 raise OSError('rm: ' + str(error))
 279               print 'Failed remove with %s, retrying' % error
 280               time.sleep(5)
 281
 282         if options.recursive:
 283           for i in range(5):
 284             try:
 285               if os.path.isdir(dst):
 286                 shutil.rmtree(dst)
 287               break
 288             except OSError as error:
 289               if i == 5:
 290                 print 'Gave up.'
 291                 raise OSError('rm: ' + str(error))
 292               print 'Failed rmtree with %s, retrying' % error
 293               time.sleep(5)
 294
 295   except OSError as error:
 296     print error
 297   return 0
 298
 299
 300 def MakeZipPath(os_path, isdir, iswindows):
 301   """Changes a path into zipfile format.
 302
 303   # doctest doesn't seem to honor r'' strings, so the backslashes need to be
 304   # escaped.
 305   >>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
 306   'users/foobar/blah'
 307   >>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
 308   'tmp/tmpfoobar/something'
 309   >>> MakeZipPath('./somefile.txt', False, False)
 310   'somefile.txt'
 311   >>> MakeZipPath('somedir', True, False)
 312   'somedir/'
 313   >>> MakeZipPath('../dir/filename.txt', False, False)
 314   '../dir/filename.txt'
 315   >>> MakeZipPath('dir/../filename.txt', False, False)
 316   'filename.txt'
 317   """
 318   zip_path = os_path
 319   if iswindows:
 320     import ntpath
 321     # zipfile paths are always posix-style. They also have the drive
 322     # letter and leading slashes removed.
 323     zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
 324   if zip_path.startswith('/'):
 325     zip_path = zip_path[1:]
 326   zip_path = posixpath.normpath(zip_path)
 327   # zipfile also always appends a slash to a directory name.
 328   if isdir:
 329     zip_path += '/'
 330   return zip_path
 331
 332
 333 def OSMakeZipPath(os_path):
 334   return MakeZipPath(os_path, os.path.isdir(os_path), sys.platform == 'win32')
 335
 336
 337 def Zip(args):
 338   """A Unix style zip.
 339
 340   Compresses the listed files."""
 341   parser = optparse.OptionParser(usage='usage: zip [Options] zipfile list')
 342   parser.add_option(
 343       '-r', dest='recursive', action='store_true',
 344       default=False,
 345       help='recurse into directories')
 346   parser.add_option(
 347       '-q', dest='quiet', action='store_true',
 348       default=False,
 349       help='quiet operation')
 350   options, files = parser.parse_args(args)
 351   if len(files) < 2:
 352     parser.error('ERROR: expecting ZIPFILE and LIST.')
 353
 354   dest_zip = files[0]
 355   src_args = files[1:]
 356
 357   src_files = []
 358   for src_arg in src_args:
 359     globbed_src_args = glob.glob(src_arg)
 360     if len(globbed_src_args) == 0:
 361       if not options.quiet:
 362         print 'zip warning: name not matched: %s' % (src_arg,)
 363
 364     for src_file in globbed_src_args:
 365       src_file = os.path.normpath(src_file)
 366       src_files.append(src_file)
 367       if options.recursive and os.path.isdir(src_file):
 368         for root, dirs, files in os.walk(src_file):
 369           for dir in dirs:
 370             src_files.append(os.path.join(root, dir))
 371           for file in files:
 372             src_files.append(os.path.join(root, file))
 373
 374   zip_stream = None
 375   # zip_data represents a list of the data to be written or appended to the
 376   # zip_stream. It is a list of tuples:
 377   #   (OS file path, zip path/zip file info, and file data)
 378   # In all cases one of the |os path| or the |file data| will be None.
 379   # |os path| is None when there is no OS file to write to the archive (i.e.
 380   # the file data already existed in the archive). |file data| is None when the
 381   # file is new (never existed in the archive) or being updated.
 382   zip_data = []
 383   new_files_to_add = [OSMakeZipPath(src_file) for src_file in src_files]
 384   zip_path_to_os_path_dict = dict((new_files_to_add[i], src_files[i])
 385                                   for i in range(len(src_files)))
 386   write_mode = 'a'
 387   try:
 388     zip_stream = zipfile.ZipFile(dest_zip, 'r')
 389     files_to_update = set(new_files_to_add).intersection(
 390         set(zip_stream.namelist()))
 391     if files_to_update:
 392       # As far as I can tell, there is no way to update a zip entry using
 393       # zipfile; the best you can do is rewrite the archive.
 394       # Iterate through the zipfile to maintain file order.
 395       write_mode = 'w'
 396       for zip_path in zip_stream.namelist():
 397         if zip_path in files_to_update:
 398           os_path = zip_path_to_os_path_dict[zip_path]
 399           zip_data.append((os_path, zip_path, None))
 400           new_files_to_add.remove(zip_path)
 401         else:
 402           file_bytes = zip_stream.read(zip_path)
 403           file_info = zip_stream.getinfo(zip_path)
 404           zip_data.append((None, file_info, file_bytes))
 405   except IOError:
 406     pass
 407   finally:
 408     if zip_stream:
 409       zip_stream.close()
 410
 411   for zip_path in new_files_to_add:
 412     zip_data.append((zip_path_to_os_path_dict[zip_path], zip_path, None))
 413
 414   if not zip_data:
 415     print 'zip error: Nothing to do! (%s)' % (dest_zip,)
 416     return 1
 417
 418   try:
 419     zip_stream = zipfile.ZipFile(dest_zip, write_mode, zipfile.ZIP_DEFLATED)
 420     for os_path, file_info_or_zip_path, file_bytes in zip_data:
 421       if isinstance(file_info_or_zip_path, zipfile.ZipInfo):
 422         zip_path = file_info_or_zip_path.filename
 423       else:
 424         zip_path = file_info_or_zip_path
 425
 426       if os_path:
 427         zip_stream.write(os_path, zip_path)
 428       else:
 429         zip_stream.writestr(file_info_or_zip_path, file_bytes)
 430
 431       if not options.quiet:
 432         if zip_path in new_files_to_add:
 433           operation = 'adding'
 434         else:
 435           operation = 'updating'
 436         zip_info = zip_stream.getinfo(zip_path)
 437         if (zip_info.compress_type == zipfile.ZIP_STORED or
 438             zip_info.file_size == 0):
 439           print '  %s: %s (stored 0%%)' % (operation, zip_path)
 440         elif zip_info.compress_type == zipfile.ZIP_DEFLATED:
 441           print '  %s: %s (deflated %d%%)' % (operation, zip_path,
 442               100 - zip_info.compress_size * 100 / zip_info.file_size)
 443   finally:
 444     zip_stream.close()
 445
 446   return 0
 447
 448
 449 FuncMap = {
 450   'cp': Copy,
 451   'mkdir': Mkdir,
 452   'mv': Move,
 453   'rm': Remove,
 454   'zip': Zip,
 455 }
 456
 457
 458 if __name__ == '__main__':
 459   func = FuncMap.get(sys.argv[1])
 460   if not func:
 461     print 'Do not recognize: ' + sys.argv[1]
 462     sys.exit(1)
 463   sys.exit(func(sys.argv[2:]))