native_client_sdk/src/tools/oshelpers.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import fnmatch
   7 import glob
   8 import optparse
   9 import os
  10 import posixpath
  11 import shutil
  12 import stat
  13 import sys
  14 import time
  15 import zipfile
  16
  17 if sys.version_info < (2, 6, 0):
  18   sys.stderr.write("python 2.6 or later is required run this script\n")
  19   sys.exit(1)
  20
  21
  22 def IncludeFiles(filters, files):
  23   """Filter files based on inclusion lists
  24
  25   Return a list of files which match and of the Unix shell-style wildcards
  26   provided, or return all the files if no filter is provided."""
  27   if not filters:
  28     return files
  29   match = set()
  30   for file_filter in filters:
  31     match |= set(fnmatch.filter(files, file_filter))
  32   return [name for name in files if name in match]
  33
  34
  35 def ExcludeFiles(filters, files):
  36   """Filter files based on exclusions lists
  37
  38   Return a list of files which do not match any of the Unix shell-style
  39   wildcards provided, or return all the files if no filter is provided."""
  40   if not filters:
  41     return files
  42   match = set()
  43   for file_filter in filters:
  44     excludes = set(fnmatch.filter(files, file_filter))
  45     match |= excludes
  46   return [name for name in files if name not in match]
  47
  48
  49 def CopyPath(options, src, dst):
  50   """CopyPath from src to dst
  51
  52   Copy a fully specified src to a fully specified dst.  If src and dst are
  53   both files, the dst file is removed first to prevent error.  If and include
  54   or exclude list are provided, the destination is first matched against that
  55   filter."""
  56   if options.includes:
  57     if not IncludeFiles(options.includes, [src]):
  58       return
  59
  60   if options.excludes:
  61     if not ExcludeFiles(options.excludes, [src]):
  62       return
  63
  64   if options.verbose:
  65     print 'cp %s %s' % (src, dst)
  66
  67   # If the source is a single file, copy it individually
  68   if os.path.isfile(src):
  69     # We can not copy over a directory with a file.
  70     if os.path.exists(dst):
  71       if not os.path.isfile(dst):
  72         msg = "cp: cannot overwrite non-file '%s' with file." % dst
  73         raise OSError(msg)
  74       # If the destination exists as a file, remove it before copying to avoid
  75       # 'readonly' issues.
  76       os.remove(dst)
  77
  78     # Now copy to the non-existent fully qualified target
  79     shutil.copy(src, dst)
  80     return
  81
  82   # Otherwise it's a directory, ignore it unless allowed
  83   if os.path.isdir(src):
  84     if not options.recursive:
  85       print "cp: omitting directory '%s'" % src
  86       return
  87
  88     # We can not copy over a file with a directory.
  89     if os.path.exists(dst):
  90       if not os.path.isdir(dst):
  91         msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
  92         raise OSError(msg)
  93     else:
  94       # if it didn't exist, create the directory
  95       os.makedirs(dst)
  96
  97     # Now copy all members
  98     for filename in os.listdir(src):
  99       srcfile = os.path.join(src, filename)
 100       dstfile = os.path.join(dst, filename)
 101       CopyPath(options, srcfile, dstfile)
 102   return
 103
 104
 105 def Copy(args):
 106   """A Unix cp style copy.
 107
 108   Copies multiple sources to a single destination using the normal cp
 109   semantics.  In addition, it support inclusion and exclusion filters which
 110   allows the copy to skip certain types of files."""
 111   parser = optparse.OptionParser(usage='usage: cp [Options] sources... dest')
 112   parser.add_option(
 113       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 114       default=False,
 115       help='copy directories recursively.')
 116   parser.add_option(
 117       '-v', '--verbose', dest='verbose', action='store_true',
 118       default=False,
 119       help='verbose output.')
 120   parser.add_option(
 121       '--include', dest='includes', action='append', default=[],
 122       help='include files matching this expression.')
 123   parser.add_option(
 124       '--exclude', dest='excludes', action='append', default=[],
 125       help='exclude files matching this expression.')
 126   options, files = parser.parse_args(args)
 127   if len(files) < 2:
 128     parser.error('ERROR: expecting SOURCE(s) and DEST.')
 129
 130   srcs = files[:-1]
 131   dst = files[-1]
 132
 133   src_list = []
 134   for src in srcs:
 135     files = glob.glob(src)
 136     if not files:
 137       raise OSError('cp: no such file or directory: ' + src)
 138     if files:
 139       src_list.extend(files)
 140
 141   for src in src_list:
 142     # If the destination is a directory, then append the basename of the src
 143     # to the destination.
 144     if os.path.isdir(dst):
 145       CopyPath(options, src, os.path.join(dst, os.path.basename(src)))
 146     else:
 147       CopyPath(options, src, dst)
 148
 149
 150 def Mkdir(args):
 151   """A Unix style mkdir"""
 152   parser = optparse.OptionParser(usage='usage: mkdir [Options] DIRECTORY...')
 153   parser.add_option(
 154       '-p', '--parents', dest='parents', action='store_true',
 155       default=False,
 156       help='ignore existing parents, create parents as needed.')
 157   parser.add_option(
 158       '-v', '--verbose', dest='verbose', action='store_true',
 159       default=False,
 160       help='verbose output.')
 161
 162   options, dsts = parser.parse_args(args)
 163   if len(dsts) < 1:
 164     parser.error('ERROR: expecting DIRECTORY...')
 165
 166   for dst in dsts:
 167     if options.verbose:
 168       print 'mkdir ' + dst
 169     try:
 170       os.makedirs(dst)
 171     except OSError:
 172       if os.path.isdir(dst):
 173         if options.parents:
 174           continue
 175         raise OSError('mkdir: Already exists: ' + dst)
 176       else:
 177         raise OSError('mkdir: Failed to create: ' + dst)
 178   return 0
 179
 180
 181 def MovePath(options, src, dst):
 182   """MovePath from src to dst
 183
 184   Moves the src to the dst much like the Unix style mv command, except it
 185   only handles one source at a time.  Because of possible temporary failures
 186   do to locks (such as anti-virus software on Windows), the function will retry
 187   up to five times."""
 188   # if the destination is not an existing directory, then overwrite it
 189   if os.path.isdir(dst):
 190     dst = os.path.join(dst, os.path.basename(src))
 191
 192   # If the destination exists, the remove it
 193   if os.path.exists(dst):
 194     if options.force:
 195       Remove(['-vfr', dst])
 196       if os.path.exists(dst):
 197         raise OSError('mv: FAILED TO REMOVE ' + dst)
 198     else:
 199       raise OSError('mv: already exists ' + dst)
 200   for _ in range(5):
 201     try:
 202       os.rename(src, dst)
 203       return
 204     except OSError as error:
 205       print 'Failed on %s with %s, retrying' % (src, error)
 206       time.sleep(5)
 207   print 'Gave up.'
 208   raise OSError('mv: ' + error)
 209
 210
 211 def Move(args):
 212   parser = optparse.OptionParser(usage='usage: mv [Options] sources... dest')
 213   parser.add_option(
 214       '-v', '--verbose', dest='verbose', action='store_true',
 215       default=False,
 216       help='verbose output.')
 217   parser.add_option(
 218       '-f', '--force', dest='force', action='store_true',
 219       default=False,
 220       help='force, do not error it files already exist.')
 221   options, files = parser.parse_args(args)
 222   if len(files) < 2:
 223     parser.error('ERROR: expecting SOURCE... and DEST.')
 224
 225   srcs = files[:-1]
 226   dst = files[-1]
 227
 228   if options.verbose:
 229     print 'mv %s %s' % (' '.join(srcs), dst)
 230
 231   for src in srcs:
 232     MovePath(options, src, dst)
 233   return 0
 234
 235
 236 def Remove(args):
 237   """A Unix style rm.
 238
 239   Removes the list of paths.  Because of possible temporary failures do to locks
 240   (such as anti-virus software on Windows), the function will retry up to five
 241   times."""
 242   parser = optparse.OptionParser(usage='usage: rm [Options] PATHS...')
 243   parser.add_option(
 244       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 245       default=False,
 246       help='remove directories recursively.')
 247   parser.add_option(
 248       '-v', '--verbose', dest='verbose', action='store_true',
 249       default=False,
 250       help='verbose output.')
 251   parser.add_option(
 252       '-f', '--force', dest='force', action='store_true',
 253       default=False,
 254       help='force, do not error it files does not exist.')
 255   options, files = parser.parse_args(args)
 256   if len(files) < 1:
 257     parser.error('ERROR: expecting FILE...')
 258
 259   try:
 260     for pattern in files:
 261       dst_files = glob.glob(pattern)
 262       if not dst_files:
 263         # Ignore non existing files when using force
 264         if options.force:
 265           continue
 266         raise OSError('rm: no such file or directory: ' + pattern)
 267
 268       for dst in dst_files:
 269         if options.verbose:
 270           print 'rm ' + dst
 271
 272         if os.path.isfile(dst) or os.path.islink(dst):
 273           for i in range(5):
 274             try:
 275               # Check every time, since it may have been deleted after the
 276               # previous failed attempt.
 277               if os.path.isfile(dst) or os.path.islink(dst):
 278                 os.remove(dst)
 279               break
 280             except OSError as error:
 281               if i == 5:
 282                 print 'Gave up.'
 283                 raise OSError('rm: ' + str(error))
 284               print 'Failed remove with %s, retrying' % error
 285               time.sleep(5)
 286
 287         if options.recursive:
 288           for i in range(5):
 289             try:
 290               if os.path.isdir(dst):
 291                 shutil.rmtree(dst)
 292               break
 293             except OSError as error:
 294               if i == 5:
 295                 print 'Gave up.'
 296                 raise OSError('rm: ' + str(error))
 297               print 'Failed rmtree with %s, retrying' % error
 298               time.sleep(5)
 299
 300   except OSError as error:
 301     print error
 302   return 0
 303
 304
 305 def MakeZipPath(os_path, isdir, iswindows):
 306   """Changes a path into zipfile format.
 307
 308   # doctest doesn't seem to honor r'' strings, so the backslashes need to be
 309   # escaped.
 310   >>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
 311   'users/foobar/blah'
 312   >>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
 313   'tmp/tmpfoobar/something'
 314   >>> MakeZipPath('./somefile.txt', False, False)
 315   'somefile.txt'
 316   >>> MakeZipPath('somedir', True, False)
 317   'somedir/'
 318   >>> MakeZipPath('../dir/filename.txt', False, False)
 319   '../dir/filename.txt'
 320   >>> MakeZipPath('dir/../filename.txt', False, False)
 321   'filename.txt'
 322   """
 323   zip_path = os_path
 324   if iswindows:
 325     import ntpath
 326     # zipfile paths are always posix-style. They also have the drive
 327     # letter and leading slashes removed.
 328     zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
 329   if zip_path.startswith('/'):
 330     zip_path = zip_path[1:]
 331   zip_path = posixpath.normpath(zip_path)
 332   # zipfile also always appends a slash to a directory name.
 333   if isdir:
 334     zip_path += '/'
 335   return zip_path
 336
 337
 338 def OSMakeZipPath(os_path):
 339   return MakeZipPath(os_path, os.path.isdir(os_path), sys.platform == 'win32')
 340
 341
 342 def Zip(args):
 343   """A Unix style zip.
 344
 345   Compresses the listed files."""
 346   parser = optparse.OptionParser(usage='usage: zip [Options] zipfile list')
 347   parser.add_option(
 348       '-r', dest='recursive', action='store_true',
 349       default=False,
 350       help='recurse into directories')
 351   parser.add_option(
 352       '-q', dest='quiet', action='store_true',
 353       default=False,
 354       help='quiet operation')
 355   options, files = parser.parse_args(args)
 356   if len(files) < 2:
 357     parser.error('ERROR: expecting ZIPFILE and LIST.')
 358
 359   dest_zip = files[0]
 360   src_args = files[1:]
 361
 362   src_files = []
 363   for src_arg in src_args:
 364     globbed_src_args = glob.glob(src_arg)
 365     if not globbed_src_args:
 366       if not options.quiet:
 367         print 'zip warning: name not matched: %s' % (src_arg,)
 368
 369     for src_file in globbed_src_args:
 370       src_file = os.path.normpath(src_file)
 371       src_files.append(src_file)
 372       if options.recursive and os.path.isdir(src_file):
 373         for root, dirs, files in os.walk(src_file):
 374           for dirname in dirs:
 375             src_files.append(os.path.join(root, dirname))
 376           for filename in files:
 377             src_files.append(os.path.join(root, filename))
 378
 379   zip_stream = None
 380   # zip_data represents a list of the data to be written or appended to the
 381   # zip_stream. It is a list of tuples:
 382   #   (OS file path, zip path/zip file info, and file data)
 383   # In all cases one of the |os path| or the |file data| will be None.
 384   # |os path| is None when there is no OS file to write to the archive (i.e.
 385   # the file data already existed in the archive). |file data| is None when the
 386   # file is new (never existed in the archive) or being updated.
 387   zip_data = []
 388   new_files_to_add = [OSMakeZipPath(src_file) for src_file in src_files]
 389   zip_path_to_os_path_dict = dict((new_files_to_add[i], src_files[i])
 390                                   for i in range(len(src_files)))
 391   write_mode = 'a'
 392   try:
 393     zip_stream = zipfile.ZipFile(dest_zip, 'r')
 394     files_to_update = set(new_files_to_add).intersection(
 395         set(zip_stream.namelist()))
 396     if files_to_update:
 397       # As far as I can tell, there is no way to update a zip entry using
 398       # zipfile; the best you can do is rewrite the archive.
 399       # Iterate through the zipfile to maintain file order.
 400       write_mode = 'w'
 401       for zip_path in zip_stream.namelist():
 402         if zip_path in files_to_update:
 403           os_path = zip_path_to_os_path_dict[zip_path]
 404           zip_data.append((os_path, zip_path, None))
 405           new_files_to_add.remove(zip_path)
 406         else:
 407           file_bytes = zip_stream.read(zip_path)
 408           file_info = zip_stream.getinfo(zip_path)
 409           zip_data.append((None, file_info, file_bytes))
 410   except IOError:
 411     pass
 412   finally:
 413     if zip_stream:
 414       zip_stream.close()
 415
 416   for zip_path in new_files_to_add:
 417     zip_data.append((zip_path_to_os_path_dict[zip_path], zip_path, None))
 418
 419   if not zip_data:
 420     print 'zip error: Nothing to do! (%s)' % (dest_zip,)
 421     return 1
 422
 423   try:
 424     zip_stream = zipfile.ZipFile(dest_zip, write_mode, zipfile.ZIP_DEFLATED)
 425     for os_path, file_info_or_zip_path, file_bytes in zip_data:
 426       if isinstance(file_info_or_zip_path, zipfile.ZipInfo):
 427         zip_path = file_info_or_zip_path.filename
 428       else:
 429         zip_path = file_info_or_zip_path
 430
 431       if os_path:
 432         st = os.stat(os_path)
 433         if stat.S_ISDIR(st.st_mode):
 434           # Python 2.6 on the buildbots doesn't support writing directories to
 435           # zip files. This was resolved in a later version of Python 2.6.
 436           # We'll work around it by writing an empty file with the correct
 437           # path. (This is basically what later versions do anyway.)
 438           zip_info = zipfile.ZipInfo()
 439           zip_info.filename = zip_path
 440           zip_info.date_time = time.localtime(st.st_mtime)[0:6]
 441           zip_info.compress_type = zip_stream.compression
 442           zip_info.flag_bits = 0x00
 443           zip_info.external_attr = (st[0] & 0xFFFF) << 16L
 444           zip_info.CRC = 0
 445           zip_info.compress_size = 0
 446           zip_info.file_size = 0
 447           zip_stream.writestr(zip_info, '')
 448         else:
 449           zip_stream.write(os_path, zip_path)
 450       else:
 451         zip_stream.writestr(file_info_or_zip_path, file_bytes)
 452
 453       if not options.quiet:
 454         if zip_path in new_files_to_add:
 455           operation = 'adding'
 456         else:
 457           operation = 'updating'
 458         zip_info = zip_stream.getinfo(zip_path)
 459         if (zip_info.compress_type == zipfile.ZIP_STORED or
 460             zip_info.file_size == 0):
 461           print '  %s: %s (stored 0%%)' % (operation, zip_path)
 462         elif zip_info.compress_type == zipfile.ZIP_DEFLATED:
 463           print '  %s: %s (deflated %d%%)' % (operation, zip_path,
 464               100 - zip_info.compress_size * 100 / zip_info.file_size)
 465   finally:
 466     zip_stream.close()
 467
 468   return 0
 469
 470
 471 def FindExeInPath(filename):
 472   env_path = os.environ.get('PATH', '')
 473   paths = env_path.split(os.pathsep)
 474
 475   def IsExecutableFile(path):
 476     return os.path.isfile(path) and os.access(path, os.X_OK)
 477
 478   if os.path.sep in filename:
 479     if IsExecutableFile(filename):
 480       return filename
 481
 482   for path in paths:
 483     filepath = os.path.join(path, filename)
 484     if IsExecutableFile(filepath):
 485       return os.path.abspath(os.path.join(path, filename))
 486
 487
 488 def Which(args):
 489   """A Unix style which.
 490
 491   Looks for all arguments in the PATH environment variable, and prints their
 492   path if they are executable files.
 493
 494   Note: If you pass an argument with a path to which, it will just test if it
 495   is executable, not if it is in the path.
 496   """
 497   parser = optparse.OptionParser(usage='usage: which args...')
 498   _, files = parser.parse_args(args)
 499   if not files:
 500     return 0
 501
 502   retval = 0
 503   for filename in files:
 504     fullname = FindExeInPath(filename)
 505     if fullname:
 506       print fullname
 507     else:
 508       retval = 1
 509
 510   return retval
 511
 512
 513 FuncMap = {
 514   'cp': Copy,
 515   'mkdir': Mkdir,
 516   'mv': Move,
 517   'rm': Remove,
 518   'zip': Zip,
 519   'which': Which,
 520 }
 521
 522
 523 def main(args):
 524   if not args:
 525     print 'No command specified'
 526     print 'Available commands: %s' % ' '.join(FuncMap)
 527     return 1
 528   func_name = args[0]
 529   func = FuncMap.get(func_name)
 530   if not func:
 531     print 'Do not recognize command: %s' % func_name
 532     print 'Available commands: %s' % ' '.join(FuncMap)
 533     return 1
 534   try:
 535     return func(args[1:])
 536   except KeyboardInterrupt:
 537     print '%s: interrupted' % func_name
 538     return 1
 539
 540 if __name__ == '__main__':
 541   sys.exit(main(sys.argv[1:]))