native_client_sdk/src/tools/oshelpers.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Utility script for emulating common unix commands."""
   7
   8 from __future__ import print_function
   9
  10 import fnmatch
  11 import glob
  12 import argparse
  13 import os
  14 import posixpath
  15 import shutil
  16 import stat
  17 import sys
  18 import time
  19 import zipfile
  20
  21 if sys.version_info < (2, 7, 0):
  22   sys.stderr.write("python 2.7 or later is required run this script\n")
  23   sys.exit(1)
  24
  25
  26 def IncludeFiles(filters, files):
  27   """Filter files based on inclusion lists
  28
  29   Return a list of files which match and of the Unix shell-style wildcards
  30   provided, or return all the files if no filter is provided.
  31   """
  32   if not filters:
  33     return files
  34   match = set()
  35   for file_filter in filters:
  36     match |= set(fnmatch.filter(files, file_filter))
  37   return [name for name in files if name in match]
  38
  39
  40 def ExcludeFiles(filters, files):
  41   """Filter files based on exclusions lists
  42
  43   Return a list of files which do not match any of the Unix shell-style
  44   wildcards provided, or return all the files if no filter is provided.
  45   """
  46   if not filters:
  47     return files
  48   match = set()
  49   for file_filter in filters:
  50     excludes = set(fnmatch.filter(files, file_filter))
  51     match |= excludes
  52   return [name for name in files if name not in match]
  53
  54
  55 def CopyPath(options, src, dst):
  56   """CopyPath from src to dst
  57
  58   Copy a fully specified src to a fully specified dst.  If src and dst are
  59   both files, the dst file is removed first to prevent error.  If and include
  60   or exclude list are provided, the destination is first matched against that
  61   filter.
  62   """
  63   if options.includes:
  64     if not IncludeFiles(options.includes, [src]):
  65       return
  66
  67   if options.excludes:
  68     if not ExcludeFiles(options.excludes, [src]):
  69       return
  70
  71   if options.verbose:
  72     print('cp %s %s' % (src, dst))
  73
  74   # If the source is a single file, copy it individually
  75   if os.path.isfile(src):
  76     # We can not copy over a directory with a file.
  77     if os.path.exists(dst):
  78       if not os.path.isfile(dst):
  79         msg = "cp: cannot overwrite non-file '%s' with file." % dst
  80         raise OSError(msg)
  81       # If the destination exists as a file, remove it before copying to avoid
  82       # 'readonly' issues.
  83       os.remove(dst)
  84
  85     # Now copy to the non-existent fully qualified target
  86     shutil.copy(src, dst)
  87     return
  88
  89   # Otherwise it's a directory, ignore it unless allowed
  90   if os.path.isdir(src):
  91     if not options.recursive:
  92       print("cp: omitting directory '%s'" % src)
  93       return
  94
  95     # We can not copy over a file with a directory.
  96     if os.path.exists(dst):
  97       if not os.path.isdir(dst):
  98         msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
  99         raise OSError(msg)
 100     else:
 101       # if it didn't exist, create the directory
 102       os.makedirs(dst)
 103
 104     # Now copy all members
 105     for filename in os.listdir(src):
 106       srcfile = os.path.join(src, filename)
 107       dstfile = os.path.join(dst, filename)
 108       CopyPath(options, srcfile, dstfile)
 109   return
 110
 111
 112 def Copy(args):
 113   """A Unix cp style copy.
 114
 115   Copies multiple sources to a single destination using the normal cp
 116   semantics.  In addition, it support inclusion and exclusion filters which
 117   allows the copy to skip certain types of files.
 118   """
 119   parser = argparse.ArgumentParser(usage='cp [Options] sources... dest',
 120                                    description=Copy.__doc__)
 121   parser.add_argument(
 122       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 123       default=False,
 124       help='copy directories recursively.')
 125   parser.add_argument(
 126       '-v', '--verbose', dest='verbose', action='store_true',
 127       default=False,
 128       help='verbose output.')
 129   parser.add_argument(
 130       '--include', dest='includes', action='append', default=[],
 131       help='include files matching this expression.')
 132   parser.add_argument(
 133       '--exclude', dest='excludes', action='append', default=[],
 134       help='exclude files matching this expression.')
 135   parser.add_argument('srcs', nargs='+', help='files to copy')
 136   parser.add_argument('dest', help='destination')
 137
 138   options = parser.parse_args(args)
 139
 140   src_list = []
 141   for src in options.srcs:
 142     files = glob.glob(src)
 143     if not files:
 144       raise OSError('cp: no such file or directory: ' + src)
 145     if files:
 146       src_list.extend(files)
 147
 148   for src in src_list:
 149     # If the destination is a directory, then append the basename of the src
 150     # to the destination.
 151     if os.path.isdir(options.dest):
 152       CopyPath(options, src, os.path.join(options.dest, os.path.basename(src)))
 153     else:
 154       CopyPath(options, src, options.dest)
 155
 156
 157 def Mkdir(args):
 158   """A Unix style mkdir."""
 159   parser = argparse.ArgumentParser(usage='mkdir [Options] DIRECTORY...',
 160                                    description=Mkdir.__doc__)
 161   parser.add_argument(
 162       '-p', '--parents', dest='parents', action='store_true',
 163       default=False,
 164       help='ignore existing parents, create parents as needed.')
 165   parser.add_argument(
 166       '-v', '--verbose', dest='verbose', action='store_true',
 167       default=False,
 168       help='verbose output.')
 169   parser.add_argument('dirs', nargs='+', help='directory(s) to create')
 170
 171   options = parser.parse_args(args)
 172
 173   for dst in options.dirs:
 174     if options.verbose:
 175       print('mkdir %s' % dst)
 176     try:
 177       os.makedirs(dst)
 178     except OSError:
 179       if os.path.isdir(dst):
 180         if options.parents:
 181           continue
 182         raise OSError('mkdir: Already exists: ' + dst)
 183       else:
 184         raise OSError('mkdir: Failed to create: ' + dst)
 185   return 0
 186
 187
 188 def MovePath(options, src, dst):
 189   """MovePath from src to dst.
 190
 191   Moves the src to the dst much like the Unix style mv command, except it
 192   only handles one source at a time.  Because of possible temporary failures
 193   do to locks (such as anti-virus software on Windows), the function will retry
 194   up to five times.
 195   """
 196   # if the destination is not an existing directory, then overwrite it
 197   if os.path.isdir(dst):
 198     dst = os.path.join(dst, os.path.basename(src))
 199
 200   # If the destination exists, the remove it
 201   if os.path.exists(dst):
 202     if options.force:
 203       Remove(['-vfr', dst])
 204       if os.path.exists(dst):
 205         raise OSError('mv: FAILED TO REMOVE ' + dst)
 206     else:
 207       raise OSError('mv: already exists ' + dst)
 208   for _ in range(5):
 209     try:
 210       os.rename(src, dst)
 211       break
 212     except OSError as error:
 213       print('Failed on %s with %s, retrying' % (src, error))
 214       time.sleep(5)
 215   else:
 216     print('Gave up.')
 217     raise OSError('mv: ' + error)
 218
 219
 220 def Move(args):
 221   """A Unix style mv."""
 222
 223   parser = argparse.ArgumentParser(usage='mv [Options] sources... dest',
 224                                    description=Move.__doc__)
 225   parser.add_argument(
 226       '-v', '--verbose', dest='verbose', action='store_true',
 227       default=False,
 228       help='verbose output.')
 229   parser.add_argument(
 230       '-f', '--force', dest='force', action='store_true',
 231       default=False,
 232       help='force, do not error it files already exist.')
 233   parser.add_argument('srcs', nargs='+')
 234   parser.add_argument('dest')
 235
 236   options = parser.parse_args(args)
 237
 238   if options.verbose:
 239     print('mv %s %s' % (' '.join(options.srcs), options.dest))
 240
 241   for src in options.srcs:
 242     MovePath(options, src, options.dest)
 243   return 0
 244
 245
 246 def Remove(args):
 247   """A Unix style rm.
 248
 249   Removes the list of paths.  Because of possible temporary failures do to locks
 250   (such as anti-virus software on Windows), the function will retry up to five
 251   times.
 252   """
 253   parser = argparse.ArgumentParser(usage='rm [Options] PATHS...',
 254                                    description=Remove.__doc__)
 255   parser.add_argument(
 256       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 257       default=False,
 258       help='remove directories recursively.')
 259   parser.add_argument(
 260       '-v', '--verbose', dest='verbose', action='store_true',
 261       default=False,
 262       help='verbose output.')
 263   parser.add_argument(
 264       '-f', '--force', dest='force', action='store_true',
 265       default=False,
 266       help='force, do not error it files does not exist.')
 267   parser.add_argument('files', nargs='+')
 268   options = parser.parse_args(args)
 269
 270   try:
 271     for pattern in options.files:
 272       dst_files = glob.glob(pattern)
 273       if not dst_files:
 274         # Ignore non existing files when using force
 275         if options.force:
 276           continue
 277         raise OSError('rm: no such file or directory: ' + pattern)
 278
 279       for dst in dst_files:
 280         if options.verbose:
 281           print('rm ' + dst)
 282
 283         if os.path.isfile(dst) or os.path.islink(dst):
 284           for _ in range(5):
 285             try:
 286               # Check every time, since it may have been deleted after the
 287               # previous failed attempt.
 288               if os.path.isfile(dst) or os.path.islink(dst):
 289                 os.remove(dst)
 290               break
 291             except OSError as error:
 292               print('Failed remove with %s, retrying' % error)
 293               time.sleep(5)
 294           else:
 295             print('Gave up.')
 296             raise OSError('rm: ' + str(error))
 297
 298         if options.recursive:
 299           for _ in range(5):
 300             try:
 301               if os.path.isdir(dst):
 302                 shutil.rmtree(dst)
 303               break
 304             except OSError as error:
 305               print('Failed rmtree with %s, retrying' % error)
 306               time.sleep(5)
 307           else:
 308             print('Gave up.')
 309             raise OSError('rm: ' + str(error))
 310
 311   except OSError as error:
 312     print(error)
 313
 314   return 0
 315
 316
 317 def MakeZipPath(os_path, isdir, iswindows):
 318   """Changes a path into zipfile format.
 319
 320   # doctest doesn't seem to honor r'' strings, so the backslashes need to be
 321   # escaped.
 322   >>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
 323   'users/foobar/blah'
 324   >>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
 325   'tmp/tmpfoobar/something'
 326   >>> MakeZipPath('./somefile.txt', False, False)
 327   'somefile.txt'
 328   >>> MakeZipPath('somedir', True, False)
 329   'somedir/'
 330   >>> MakeZipPath('../dir/filename.txt', False, False)
 331   '../dir/filename.txt'
 332   >>> MakeZipPath('dir/../filename.txt', False, False)
 333   'filename.txt'
 334   """
 335   zip_path = os_path
 336   if iswindows:
 337     import ntpath
 338     # zipfile paths are always posix-style. They also have the drive
 339     # letter and leading slashes removed.
 340     zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
 341   if zip_path.startswith('/'):
 342     zip_path = zip_path[1:]
 343   zip_path = posixpath.normpath(zip_path)
 344   # zipfile also always appends a slash to a directory name.
 345   if isdir:
 346     zip_path += '/'
 347   return zip_path
 348
 349
 350 def OSMakeZipPath(os_path):
 351   return MakeZipPath(os_path, os.path.isdir(os_path), sys.platform == 'win32')
 352
 353
 354 def Zip(args):
 355   """A Unix style zip.
 356
 357   Compresses the listed files.
 358   """
 359   parser = argparse.ArgumentParser(description=Zip.__doc__)
 360   parser.add_argument(
 361       '-r', dest='recursive', action='store_true',
 362       default=False,
 363       help='recurse into directories')
 364   parser.add_argument(
 365       '-q', dest='quiet', action='store_true',
 366       default=False,
 367       help='quiet operation')
 368   parser.add_argument('zipfile')
 369   parser.add_argument('filenames', nargs='+')
 370   options = parser.parse_args(args)
 371
 372   src_files = []
 373   for filename in options.filenames:
 374     globbed_src_args = glob.glob(filename)
 375     if not globbed_src_args:
 376       if not options.quiet:
 377         print('zip warning: name not matched: %s' % filename)
 378
 379     for src_file in globbed_src_args:
 380       src_file = os.path.normpath(src_file)
 381       src_files.append(src_file)
 382       if options.recursive and os.path.isdir(src_file):
 383         for root, dirs, files in os.walk(src_file):
 384           for dirname in dirs:
 385             src_files.append(os.path.join(root, dirname))
 386           for filename in files:
 387             src_files.append(os.path.join(root, filename))
 388
 389   # zip_data represents a list of the data to be written or appended to the
 390   # zip_stream. It is a list of tuples:
 391   #   (OS file path, zip path/zip file info, and file data)
 392   # In all cases one of the |os path| or the |file data| will be None.
 393   # |os path| is None when there is no OS file to write to the archive (i.e.
 394   # the file data already existed in the archive). |file data| is None when the
 395   # file is new (never existed in the archive) or being updated.
 396   zip_data = []
 397   new_files_to_add = [OSMakeZipPath(src_file) for src_file in src_files]
 398   zip_path_to_os_path_dict = dict((new_files_to_add[i], src_files[i])
 399                                   for i in range(len(src_files)))
 400   write_mode = 'a'
 401   if os.path.exists(options.zipfile):
 402     with zipfile.ZipFile(options.zipfile, 'r') as zip_stream:
 403       try:
 404         files_to_update = set(new_files_to_add).intersection(
 405             set(zip_stream.namelist()))
 406         if files_to_update:
 407           # As far as I can tell, there is no way to update a zip entry using
 408           # zipfile; the best you can do is rewrite the archive.
 409           # Iterate through the zipfile to maintain file order.
 410           write_mode = 'w'
 411           for zip_path in zip_stream.namelist():
 412             if zip_path in files_to_update:
 413               os_path = zip_path_to_os_path_dict[zip_path]
 414               zip_data.append((os_path, zip_path, None))
 415               new_files_to_add.remove(zip_path)
 416             else:
 417               file_bytes = zip_stream.read(zip_path)
 418               file_info = zip_stream.getinfo(zip_path)
 419               zip_data.append((None, file_info, file_bytes))
 420       except IOError:
 421         pass
 422
 423   for zip_path in new_files_to_add:
 424     zip_data.append((zip_path_to_os_path_dict[zip_path], zip_path, None))
 425
 426   if not zip_data:
 427     print('zip error: Nothing to do! (%s)' % options.zipfile)
 428     return 1
 429
 430   with zipfile.ZipFile(options.zipfile, write_mode,
 431                        zipfile.ZIP_DEFLATED) as zip_stream:
 432     for os_path, file_info_or_zip_path, file_bytes in zip_data:
 433       if isinstance(file_info_or_zip_path, zipfile.ZipInfo):
 434         zip_path = file_info_or_zip_path.filename
 435       else:
 436         zip_path = file_info_or_zip_path
 437
 438       if os_path:
 439         st = os.stat(os_path)
 440         if stat.S_ISDIR(st.st_mode):
 441           # Python 2.6 on the buildbots doesn't support writing directories to
 442           # zip files. This was resolved in a later version of Python 2.6.
 443           # We'll work around it by writing an empty file with the correct
 444           # path. (This is basically what later versions do anyway.)
 445           zip_info = zipfile.ZipInfo()
 446           zip_info.filename = zip_path
 447           zip_info.date_time = time.localtime(st.st_mtime)[0:6]
 448           zip_info.compress_type = zip_stream.compression
 449           zip_info.flag_bits = 0x00
 450           zip_info.external_attr = (st[0] & 0xFFFF) << 16L
 451           zip_info.CRC = 0
 452           zip_info.compress_size = 0
 453           zip_info.file_size = 0
 454           zip_stream.writestr(zip_info, '')
 455         else:
 456           zip_stream.write(os_path, zip_path)
 457       else:
 458         zip_stream.writestr(file_info_or_zip_path, file_bytes)
 459
 460       if not options.quiet:
 461         if zip_path in new_files_to_add:
 462           operation = 'adding'
 463         else:
 464           operation = 'updating'
 465         zip_info = zip_stream.getinfo(zip_path)
 466         if (zip_info.compress_type == zipfile.ZIP_STORED or
 467             zip_info.file_size == 0):
 468           print('  %s: %s (stored 0%%)' % (operation, zip_path))
 469         elif zip_info.compress_type == zipfile.ZIP_DEFLATED:
 470           print('  %s: %s (deflated %d%%)' % (operation, zip_path,
 471               100 - zip_info.compress_size * 100 / zip_info.file_size))
 472
 473   return 0
 474
 475
 476 def FindExeInPath(filename):
 477   env_path = os.environ.get('PATH', '')
 478   paths = env_path.split(os.pathsep)
 479
 480   def IsExecutableFile(path):
 481     return os.path.isfile(path) and os.access(path, os.X_OK)
 482
 483   if os.path.sep in filename:
 484     if IsExecutableFile(filename):
 485       return filename
 486
 487   for path in paths:
 488     filepath = os.path.join(path, filename)
 489     if IsExecutableFile(filepath):
 490       return os.path.abspath(os.path.join(path, filename))
 491
 492
 493 def Which(args):
 494   """A Unix style which.
 495
 496   Looks for all arguments in the PATH environment variable, and prints their
 497   path if they are executable files.
 498
 499   Note: If you pass an argument with a path to which, it will just test if it
 500   is executable, not if it is in the path.
 501   """
 502   parser = argparse.ArgumentParser(description=Which.__doc__)
 503   parser.add_argument('files', nargs='+')
 504   options = parser.parse_args(args)
 505
 506   retval = 0
 507   for filename in options.files:
 508     fullname = FindExeInPath(filename)
 509     if fullname:
 510       print(fullname)
 511     else:
 512       retval = 1
 513
 514   return retval
 515
 516
 517 FuncMap = {
 518   'cp': Copy,
 519   'mkdir': Mkdir,
 520   'mv': Move,
 521   'rm': Remove,
 522   'zip': Zip,
 523   'which': Which,
 524 }
 525
 526
 527 def main(args):
 528   if not args:
 529     print('No command specified')
 530     print('Available commands: %s' % ' '.join(FuncMap))
 531     return 1
 532   func_name = args[0]
 533   func = FuncMap.get(func_name)
 534   if not func:
 535     print('Do not recognize command: %s' % func_name)
 536     print('Available commands: %s' % ' '.join(FuncMap))
 537     return 1
 538   try:
 539     return func(args[1:])
 540   except KeyboardInterrupt:
 541     print('%s: interrupted' % func_name)
 542     return 1
 543
 544 if __name__ == '__main__':
 545   sys.exit(main(sys.argv[1:]))