native_client_sdk/src/tools/oshelpers.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import fnmatch
   7 import glob
   8 import optparse
   9 import os
  10 import posixpath
  11 import shutil
  12 import stat
  13 import sys
  14 import time
  15 import zipfile
  16
  17 if sys.version_info < (2, 6, 0):
  18   sys.stderr.write("python 2.6 or later is required run this script\n")
  19   sys.exit(1)
  20
  21
  22 def IncludeFiles(filters, files):
  23   """Filter files based on inclusion lists
  24
  25   Return a list of files which match and of the Unix shell-style wildcards
  26   provided, or return all the files if no filter is provided."""
  27   if not filters:
  28     return files
  29   match = set()
  30   for file_filter in filters:
  31     match |= set(fnmatch.filter(files, file_filter))
  32   return [name for name in files if name in match]
  33
  34
  35 def ExcludeFiles(filters, files):
  36   """Filter files based on exclusions lists
  37
  38   Return a list of files which do not match any of the Unix shell-style
  39   wildcards provided, or return all the files if no filter is provided."""
  40   if not filters:
  41     return files
  42   match = set()
  43   for file_filter in filters:
  44     excludes = set(fnmatch.filter(files, file_filter))
  45     match |= excludes
  46   return [name for name in files if name not in match]
  47
  48
  49 def CopyPath(options, src, dst):
  50   """CopyPath from src to dst
  51
  52   Copy a fully specified src to a fully specified dst.  If src and dst are
  53   both files, the dst file is removed first to prevent error.  If and include
  54   or exclude list are provided, the destination is first matched against that
  55   filter."""
  56   if options.includes:
  57     if not IncludeFiles(options.includes, [src]):
  58       return
  59
  60   if options.excludes:
  61     if not ExcludeFiles(options.excludes, [src]):
  62       return
  63
  64   if options.verbose:
  65     print 'cp %s %s' % (src, dst)
  66
  67   # If the source is a single file, copy it individually
  68   if os.path.isfile(src):
  69     # We can not copy over a directory with a file.
  70     if os.path.exists(dst):
  71       if not os.path.isfile(dst):
  72         msg = "cp: cannot overwrite non-file '%s' with file." % dst
  73         raise OSError(msg)
  74       # If the destination exists as a file, remove it before copying to avoid
  75       # 'readonly' issues.
  76       os.remove(dst)
  77
  78     # Now copy to the non-existent fully qualified target
  79     shutil.copy(src, dst)
  80     return
  81
  82   # Otherwise it's a directory, ignore it unless allowed
  83   if os.path.isdir(src):
  84     if not options.recursive:
  85       print "cp: omitting directory '%s'" % src
  86       return
  87
  88     # We can not copy over a file with a directory.
  89     if os.path.exists(dst):
  90       if not os.path.isdir(dst):
  91         msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
  92         raise OSError(msg)
  93     else:
  94       # if it didn't exist, create the directory
  95       os.makedirs(dst)
  96
  97     # Now copy all members
  98     for filename in os.listdir(src):
  99       srcfile = os.path.join(src, filename)
 100       dstfile = os.path.join(dst, filename)
 101       CopyPath(options, srcfile, dstfile)
 102   return
 103
 104
 105 def Copy(args):
 106   """A Unix cp style copy.
 107
 108   Copies multiple sources to a single destination using the normal cp
 109   semantics.  In addition, it support inclusion and exclusion filters which
 110   allows the copy to skip certain types of files."""
 111   parser = optparse.OptionParser(usage='usage: cp [Options] souces... dest')
 112   parser.add_option(
 113       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 114       default=False,
 115       help='copy directories recursively.')
 116   parser.add_option(
 117       '-v', '--verbose', dest='verbose', action='store_true',
 118       default=False,
 119       help='verbose output.')
 120   parser.add_option(
 121       '--include', dest='includes', action='append', default=[],
 122       help='include files matching this expression.')
 123   parser.add_option(
 124       '--exclude', dest='excludes', action='append', default=[],
 125       help='exclude files matching this expression.')
 126   options, files = parser.parse_args(args)
 127   if len(files) < 2:
 128     parser.error('ERROR: expecting SOURCE(s) and DEST.')
 129
 130   srcs = files[:-1]
 131   dst = files[-1]
 132
 133   src_list = []
 134   for src in srcs:
 135     files = glob.glob(src)
 136     if len(files) == 0:
 137       raise OSError('cp: no such file or directory: ' + src)
 138     if files:
 139       src_list.extend(files)
 140
 141   for src in src_list:
 142     # If the destination is a directory, then append the basename of the src
 143     # to the destination.
 144     if os.path.isdir(dst):
 145       CopyPath(options, src, os.path.join(dst, os.path.basename(src)))
 146     else:
 147       CopyPath(options, src, dst)
 148
 149
 150 def Mkdir(args):
 151   """A Unix style mkdir"""
 152   parser = optparse.OptionParser(usage='usage: mkdir [Options] DIRECTORY...')
 153   parser.add_option(
 154       '-p', '--parents', dest='parents', action='store_true',
 155       default=False,
 156       help='ignore existing parents, create parents as needed.')
 157   parser.add_option(
 158       '-v', '--verbose', dest='verbose', action='store_true',
 159       default=False,
 160       help='verbose output.')
 161
 162   options, dsts = parser.parse_args(args)
 163   if len(dsts) < 1:
 164     parser.error('ERROR: expecting DIRECTORY...')
 165
 166   for dst in dsts:
 167     if options.verbose:
 168       print 'mkdir ' + dst
 169     try:
 170       os.makedirs(dst)
 171     except OSError:
 172       if os.path.isdir(dst):
 173         if options.parents:
 174           continue
 175         raise OSError('mkdir: Already exsists: ' + dst)
 176       else:
 177         raise OSError('mkdir: Failed to create: ' + dst)
 178   return 0
 179
 180
 181 def MovePath(options, src, dst):
 182   """MovePath from src to dst
 183
 184   Moves the src to the dst much like the Unix style mv command, except it
 185   only handles one source at a time.  Because of possible temporary failures
 186   do to locks (such as anti-virus software on Windows), the function will retry
 187   up to five times."""
 188   # if the destination is not an existing directory, then overwrite it
 189   if os.path.isdir(dst):
 190     dst = os.path.join(dst, os.path.basename(src))
 191
 192   # If the destination exists, the remove it
 193   if os.path.exists(dst):
 194     if options.force:
 195       Remove(['-vfr', dst])
 196       if os.path.exists(dst):
 197         raise OSError('mv: FAILED TO REMOVE ' + dst)
 198     else:
 199       raise OSError('mv: already exists ' + dst)
 200   for _ in range(5):
 201     try:
 202       os.rename(src, dst)
 203       return
 204     except OSError as error:
 205       print 'Failed on %s with %s, retrying' % (src, error)
 206       time.sleep(5)
 207   print 'Gave up.'
 208   raise OSError('mv: ' + error)
 209
 210
 211 def Move(args):
 212   parser = optparse.OptionParser(usage='usage: mv [Options] souces... dest')
 213   parser.add_option(
 214       '-v', '--verbose', dest='verbose', action='store_true',
 215       default=False,
 216       help='verbose output.')
 217   parser.add_option(
 218       '-f', '--force', dest='force', action='store_true',
 219       default=False,
 220       help='force, do not error it files already exist.')
 221   options, files = parser.parse_args(args)
 222   if len(files) < 2:
 223     parser.error('ERROR: expecting SOURCE... and DEST.')
 224
 225   srcs = files[:-1]
 226   dst = files[-1]
 227
 228   if options.verbose:
 229     print 'mv %s %s' % (' '.join(srcs), dst)
 230
 231   for src in srcs:
 232     MovePath(options, src, dst)
 233   return 0
 234
 235
 236 def Remove(args):
 237   """A Unix style rm.
 238
 239   Removes the list of paths.  Because of possible temporary failures do to locks
 240   (such as anti-virus software on Windows), the function will retry up to five
 241   times."""
 242   parser = optparse.OptionParser(usage='usage: rm [Options] PATHS...')
 243   parser.add_option(
 244       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 245       default=False,
 246       help='remove directories recursively.')
 247   parser.add_option(
 248       '-v', '--verbose', dest='verbose', action='store_true',
 249       default=False,
 250       help='verbose output.')
 251   parser.add_option(
 252       '-f', '--force', dest='force', action='store_true',
 253       default=False,
 254       help='force, do not error it files does not exist.')
 255   options, files = parser.parse_args(args)
 256   if len(files) < 1:
 257     parser.error('ERROR: expecting FILE...')
 258
 259   try:
 260     for pattern in files:
 261       dst_files = glob.glob(pattern)
 262       # Ignore non existing files when using force
 263       if len(dst_files) == 0 and options.force:
 264         print "rm: Skipping " + pattern
 265         continue
 266       elif len(dst_files) == 0:
 267         raise OSError('rm: no such file or directory: ' + pattern)
 268
 269       for dst in dst_files:
 270         if options.verbose:
 271           print 'rm ' + dst
 272
 273         if os.path.isfile(dst) or os.path.islink(dst):
 274           for i in range(5):
 275             try:
 276               # Check every time, since it may have been deleted after the
 277               # previous failed attempt.
 278               if os.path.isfile(dst) or os.path.islink(dst):
 279                 os.remove(dst)
 280               break
 281             except OSError as error:
 282               if i == 5:
 283                 print 'Gave up.'
 284                 raise OSError('rm: ' + str(error))
 285               print 'Failed remove with %s, retrying' % error
 286               time.sleep(5)
 287
 288         if options.recursive:
 289           for i in range(5):
 290             try:
 291               if os.path.isdir(dst):
 292                 shutil.rmtree(dst)
 293               break
 294             except OSError as error:
 295               if i == 5:
 296                 print 'Gave up.'
 297                 raise OSError('rm: ' + str(error))
 298               print 'Failed rmtree with %s, retrying' % error
 299               time.sleep(5)
 300
 301   except OSError as error:
 302     print error
 303   return 0
 304
 305
 306 def MakeZipPath(os_path, isdir, iswindows):
 307   """Changes a path into zipfile format.
 308
 309   # doctest doesn't seem to honor r'' strings, so the backslashes need to be
 310   # escaped.
 311   >>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
 312   'users/foobar/blah'
 313   >>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
 314   'tmp/tmpfoobar/something'
 315   >>> MakeZipPath('./somefile.txt', False, False)
 316   'somefile.txt'
 317   >>> MakeZipPath('somedir', True, False)
 318   'somedir/'
 319   >>> MakeZipPath('../dir/filename.txt', False, False)
 320   '../dir/filename.txt'
 321   >>> MakeZipPath('dir/../filename.txt', False, False)
 322   'filename.txt'
 323   """
 324   zip_path = os_path
 325   if iswindows:
 326     import ntpath
 327     # zipfile paths are always posix-style. They also have the drive
 328     # letter and leading slashes removed.
 329     zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
 330   if zip_path.startswith('/'):
 331     zip_path = zip_path[1:]
 332   zip_path = posixpath.normpath(zip_path)
 333   # zipfile also always appends a slash to a directory name.
 334   if isdir:
 335     zip_path += '/'
 336   return zip_path
 337
 338
 339 def OSMakeZipPath(os_path):
 340   return MakeZipPath(os_path, os.path.isdir(os_path), sys.platform == 'win32')
 341
 342
 343 def Zip(args):
 344   """A Unix style zip.
 345
 346   Compresses the listed files."""
 347   parser = optparse.OptionParser(usage='usage: zip [Options] zipfile list')
 348   parser.add_option(
 349       '-r', dest='recursive', action='store_true',
 350       default=False,
 351       help='recurse into directories')
 352   parser.add_option(
 353       '-q', dest='quiet', action='store_true',
 354       default=False,
 355       help='quiet operation')
 356   options, files = parser.parse_args(args)
 357   if len(files) < 2:
 358     parser.error('ERROR: expecting ZIPFILE and LIST.')
 359
 360   dest_zip = files[0]
 361   src_args = files[1:]
 362
 363   src_files = []
 364   for src_arg in src_args:
 365     globbed_src_args = glob.glob(src_arg)
 366     if len(globbed_src_args) == 0:
 367       if not options.quiet:
 368         print 'zip warning: name not matched: %s' % (src_arg,)
 369
 370     for src_file in globbed_src_args:
 371       src_file = os.path.normpath(src_file)
 372       src_files.append(src_file)
 373       if options.recursive and os.path.isdir(src_file):
 374         for root, dirs, files in os.walk(src_file):
 375           for dirname in dirs:
 376             src_files.append(os.path.join(root, dirname))
 377           for filename in files:
 378             src_files.append(os.path.join(root, filename))
 379
 380   zip_stream = None
 381   # zip_data represents a list of the data to be written or appended to the
 382   # zip_stream. It is a list of tuples:
 383   #   (OS file path, zip path/zip file info, and file data)
 384   # In all cases one of the |os path| or the |file data| will be None.
 385   # |os path| is None when there is no OS file to write to the archive (i.e.
 386   # the file data already existed in the archive). |file data| is None when the
 387   # file is new (never existed in the archive) or being updated.
 388   zip_data = []
 389   new_files_to_add = [OSMakeZipPath(src_file) for src_file in src_files]
 390   zip_path_to_os_path_dict = dict((new_files_to_add[i], src_files[i])
 391                                   for i in range(len(src_files)))
 392   write_mode = 'a'
 393   try:
 394     zip_stream = zipfile.ZipFile(dest_zip, 'r')
 395     files_to_update = set(new_files_to_add).intersection(
 396         set(zip_stream.namelist()))
 397     if files_to_update:
 398       # As far as I can tell, there is no way to update a zip entry using
 399       # zipfile; the best you can do is rewrite the archive.
 400       # Iterate through the zipfile to maintain file order.
 401       write_mode = 'w'
 402       for zip_path in zip_stream.namelist():
 403         if zip_path in files_to_update:
 404           os_path = zip_path_to_os_path_dict[zip_path]
 405           zip_data.append((os_path, zip_path, None))
 406           new_files_to_add.remove(zip_path)
 407         else:
 408           file_bytes = zip_stream.read(zip_path)
 409           file_info = zip_stream.getinfo(zip_path)
 410           zip_data.append((None, file_info, file_bytes))
 411   except IOError:
 412     pass
 413   finally:
 414     if zip_stream:
 415       zip_stream.close()
 416
 417   for zip_path in new_files_to_add:
 418     zip_data.append((zip_path_to_os_path_dict[zip_path], zip_path, None))
 419
 420   if not zip_data:
 421     print 'zip error: Nothing to do! (%s)' % (dest_zip,)
 422     return 1
 423
 424   try:
 425     zip_stream = zipfile.ZipFile(dest_zip, write_mode, zipfile.ZIP_DEFLATED)
 426     for os_path, file_info_or_zip_path, file_bytes in zip_data:
 427       if isinstance(file_info_or_zip_path, zipfile.ZipInfo):
 428         zip_path = file_info_or_zip_path.filename
 429       else:
 430         zip_path = file_info_or_zip_path
 431
 432       if os_path:
 433         st = os.stat(os_path)
 434         if stat.S_ISDIR(st.st_mode):
 435           # Python 2.6 on the buildbots doesn't support writing directories to
 436           # zip files. This was resolved in a later version of Python 2.6.
 437           # We'll work around it by writing an empty file with the correct
 438           # path. (This is basically what later versions do anyway.)
 439           zip_info = zipfile.ZipInfo()
 440           zip_info.filename = zip_path
 441           zip_info.date_time = time.localtime(st.st_mtime)[0:6]
 442           zip_info.compress_type = zip_stream.compression
 443           zip_info.flag_bits = 0x00
 444           zip_info.external_attr = (st[0] & 0xFFFF) << 16L
 445           zip_info.CRC = 0
 446           zip_info.compress_size = 0
 447           zip_info.file_size = 0
 448           zip_stream.writestr(zip_info, '')
 449         else:
 450           zip_stream.write(os_path, zip_path)
 451       else:
 452         zip_stream.writestr(file_info_or_zip_path, file_bytes)
 453
 454       if not options.quiet:
 455         if zip_path in new_files_to_add:
 456           operation = 'adding'
 457         else:
 458           operation = 'updating'
 459         zip_info = zip_stream.getinfo(zip_path)
 460         if (zip_info.compress_type == zipfile.ZIP_STORED or
 461             zip_info.file_size == 0):
 462           print '  %s: %s (stored 0%%)' % (operation, zip_path)
 463         elif zip_info.compress_type == zipfile.ZIP_DEFLATED:
 464           print '  %s: %s (deflated %d%%)' % (operation, zip_path,
 465               100 - zip_info.compress_size * 100 / zip_info.file_size)
 466   finally:
 467     zip_stream.close()
 468
 469   return 0
 470
 471
 472 FuncMap = {
 473   'cp': Copy,
 474   'mkdir': Mkdir,
 475   'mv': Move,
 476   'rm': Remove,
 477   'zip': Zip,
 478 }
 479
 480
 481 def main(args):
 482   if not args:
 483     print 'No command specified'
 484     print 'Available commands: %s' % ' '.join(FuncMap)
 485     return 1
 486   func = FuncMap.get(args[0])
 487   if not func:
 488     print 'Do not recognize command: ' + args[0]
 489     print 'Available commands: %s' % ' '.join(FuncMap)
 490     return 1
 491   return func(args[1:])
 492
 493 if __name__ == '__main__':
 494   sys.exit(main(sys.argv[1:]))