native_client_sdk/src/tools/oshelpers.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Utility script for emulating common unix commands."""
   7
   8 from __future__ import print_function
   9
  10 import argparse
  11 import fnmatch
  12 import glob
  13 import os
  14 import posixpath
  15 import shutil
  16 import stat
  17 import subprocess
  18 import sys
  19 import time
  20 import zipfile
  21
  22 if sys.version_info < (2, 7, 0):
  23   sys.stderr.write("python 2.7 or later is required run this script\n")
  24   sys.exit(1)
  25
  26
  27 def IncludeFiles(filters, files):
  28   """Filter files based on inclusion lists
  29
  30   Return a list of files which match and of the Unix shell-style wildcards
  31   provided, or return all the files if no filter is provided.
  32   """
  33   if not filters:
  34     return files
  35   match = set()
  36   for file_filter in filters:
  37     match |= set(fnmatch.filter(files, file_filter))
  38   return [name for name in files if name in match]
  39
  40
  41 def ExcludeFiles(filters, files):
  42   """Filter files based on exclusions lists
  43
  44   Return a list of files which do not match any of the Unix shell-style
  45   wildcards provided, or return all the files if no filter is provided.
  46   """
  47   if not filters:
  48     return files
  49   match = set()
  50   for file_filter in filters:
  51     excludes = set(fnmatch.filter(files, file_filter))
  52     match |= excludes
  53   return [name for name in files if name not in match]
  54
  55
  56 def CopyPath(options, src, dst):
  57   """CopyPath from src to dst
  58
  59   Copy a fully specified src to a fully specified dst.  If src and dst are
  60   both files, the dst file is removed first to prevent error.  If and include
  61   or exclude list are provided, the destination is first matched against that
  62   filter.
  63   """
  64   if options.includes:
  65     if not IncludeFiles(options.includes, [src]):
  66       return
  67
  68   if options.excludes:
  69     if not ExcludeFiles(options.excludes, [src]):
  70       return
  71
  72   if options.verbose:
  73     print('cp %s %s' % (src, dst))
  74
  75   # If the source is a single file, copy it individually
  76   if os.path.isfile(src):
  77     # We can not copy over a directory with a file.
  78     if os.path.exists(dst):
  79       if not os.path.isfile(dst):
  80         msg = "cp: cannot overwrite non-file '%s' with file." % dst
  81         raise OSError(msg)
  82       # If the destination exists as a file, remove it before copying to avoid
  83       # 'readonly' issues.
  84       os.remove(dst)
  85
  86     # Now copy to the non-existent fully qualified target
  87     shutil.copy(src, dst)
  88     return
  89
  90   # Otherwise it's a directory, ignore it unless allowed
  91   if os.path.isdir(src):
  92     if not options.recursive:
  93       print("cp: omitting directory '%s'" % src)
  94       return
  95
  96     # We can not copy over a file with a directory.
  97     if os.path.exists(dst):
  98       if not os.path.isdir(dst):
  99         msg = "cp: cannot overwrite non-directory '%s' with directory." % dst
 100         raise OSError(msg)
 101     else:
 102       # if it didn't exist, create the directory
 103       os.makedirs(dst)
 104
 105     # Now copy all members
 106     for filename in os.listdir(src):
 107       srcfile = os.path.join(src, filename)
 108       dstfile = os.path.join(dst, filename)
 109       CopyPath(options, srcfile, dstfile)
 110   return
 111
 112
 113 def Copy(args):
 114   """A Unix cp style copy.
 115
 116   Copies multiple sources to a single destination using the normal cp
 117   semantics.  In addition, it support inclusion and exclusion filters which
 118   allows the copy to skip certain types of files.
 119   """
 120   parser = argparse.ArgumentParser(usage='cp [Options] sources... dest',
 121                                    description=Copy.__doc__)
 122   parser.add_argument(
 123       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 124       default=False,
 125       help='copy directories recursively.')
 126   parser.add_argument(
 127       '-v', '--verbose', dest='verbose', action='store_true',
 128       default=False,
 129       help='verbose output.')
 130   parser.add_argument(
 131       '--include', dest='includes', action='append', default=[],
 132       help='include files matching this expression.')
 133   parser.add_argument(
 134       '--exclude', dest='excludes', action='append', default=[],
 135       help='exclude files matching this expression.')
 136   parser.add_argument('srcs', nargs='+', help='files to copy')
 137   parser.add_argument('dest', help='destination')
 138
 139   options = parser.parse_args(args)
 140
 141   src_list = []
 142   for src in options.srcs:
 143     files = glob.glob(src)
 144     if not files:
 145       raise OSError('cp: no such file or directory: ' + src)
 146     if files:
 147       src_list.extend(files)
 148
 149   for src in src_list:
 150     # If the destination is a directory, then append the basename of the src
 151     # to the destination.
 152     if os.path.isdir(options.dest):
 153       CopyPath(options, src, os.path.join(options.dest, os.path.basename(src)))
 154     else:
 155       CopyPath(options, src, options.dest)
 156
 157
 158 def Mkdir(args):
 159   """A Unix style mkdir."""
 160   parser = argparse.ArgumentParser(usage='mkdir [Options] DIRECTORY...',
 161                                    description=Mkdir.__doc__)
 162   parser.add_argument(
 163       '-p', '--parents', dest='parents', action='store_true',
 164       default=False,
 165       help='ignore existing parents, create parents as needed.')
 166   parser.add_argument(
 167       '-v', '--verbose', dest='verbose', action='store_true',
 168       default=False,
 169       help='verbose output.')
 170   parser.add_argument('dirs', nargs='+', help='directory(s) to create')
 171
 172   options = parser.parse_args(args)
 173
 174   for dst in options.dirs:
 175     if options.verbose:
 176       print('mkdir %s' % dst)
 177     try:
 178       os.makedirs(dst)
 179     except OSError:
 180       if os.path.isdir(dst):
 181         if options.parents:
 182           continue
 183         raise OSError('mkdir: Already exists: ' + dst)
 184       else:
 185         raise OSError('mkdir: Failed to create: ' + dst)
 186   return 0
 187
 188
 189 def MovePath(options, src, dst):
 190   """MovePath from src to dst.
 191
 192   Moves the src to the dst much like the Unix style mv command, except it
 193   only handles one source at a time.  Because of possible temporary failures
 194   do to locks (such as anti-virus software on Windows), the function will retry
 195   up to five times.
 196   """
 197   # if the destination is not an existing directory, then overwrite it
 198   if os.path.isdir(dst):
 199     dst = os.path.join(dst, os.path.basename(src))
 200
 201   # If the destination exists, the remove it
 202   if os.path.exists(dst):
 203     if options.force:
 204       Remove(['-vfr', dst])
 205       if os.path.exists(dst):
 206         raise OSError('mv: FAILED TO REMOVE ' + dst)
 207     else:
 208       raise OSError('mv: already exists ' + dst)
 209   for _ in range(5):
 210     try:
 211       os.rename(src, dst)
 212       break
 213     except OSError as error:
 214       print('Failed on %s with %s, retrying' % (src, error))
 215       time.sleep(5)
 216   else:
 217     print('Gave up.')
 218     raise OSError('mv: ' + error)
 219
 220
 221 def Move(args):
 222   """A Unix style mv."""
 223
 224   parser = argparse.ArgumentParser(usage='mv [Options] sources... dest',
 225                                    description=Move.__doc__)
 226   parser.add_argument(
 227       '-v', '--verbose', dest='verbose', action='store_true',
 228       default=False,
 229       help='verbose output.')
 230   parser.add_argument(
 231       '-f', '--force', dest='force', action='store_true',
 232       default=False,
 233       help='force, do not error it files already exist.')
 234   parser.add_argument('srcs', nargs='+')
 235   parser.add_argument('dest')
 236
 237   options = parser.parse_args(args)
 238
 239   if options.verbose:
 240     print('mv %s %s' % (' '.join(options.srcs), options.dest))
 241
 242   for src in options.srcs:
 243     MovePath(options, src, options.dest)
 244   return 0
 245
 246
 247 def Remove(args):
 248   """A Unix style rm.
 249
 250   Removes the list of paths.  Because of possible temporary failures do to locks
 251   (such as anti-virus software on Windows), the function will retry up to five
 252   times.
 253   """
 254   parser = argparse.ArgumentParser(usage='rm [Options] PATHS...',
 255                                    description=Remove.__doc__)
 256   parser.add_argument(
 257       '-R', '-r', '--recursive', dest='recursive', action='store_true',
 258       default=False,
 259       help='remove directories recursively.')
 260   parser.add_argument(
 261       '-v', '--verbose', dest='verbose', action='store_true',
 262       default=False,
 263       help='verbose output.')
 264   parser.add_argument(
 265       '-f', '--force', dest='force', action='store_true',
 266       default=False,
 267       help='force, do not error it files does not exist.')
 268   parser.add_argument('files', nargs='+')
 269   options = parser.parse_args(args)
 270
 271   try:
 272     for pattern in options.files:
 273       dst_files = glob.glob(pattern)
 274       if not dst_files:
 275         # Ignore non existing files when using force
 276         if options.force:
 277           continue
 278         raise OSError('rm: no such file or directory: ' + pattern)
 279
 280       for dst in dst_files:
 281         if options.verbose:
 282           print('rm ' + dst)
 283
 284         if os.path.isfile(dst) or os.path.islink(dst):
 285           for _ in range(5):
 286             try:
 287               # Check every time, since it may have been deleted after the
 288               # previous failed attempt.
 289               if os.path.isfile(dst) or os.path.islink(dst):
 290                 os.remove(dst)
 291               break
 292             except OSError as error:
 293               print('Failed remove with %s, retrying' % error)
 294               time.sleep(5)
 295           else:
 296             print('Gave up.')
 297             raise OSError('rm: ' + str(error))
 298
 299         if options.recursive:
 300           for _ in range(5):
 301             try:
 302               if os.path.isdir(dst):
 303                 if sys.platform == 'win32':
 304                   # shutil.rmtree doesn't handle junctions properly. Let's just
 305                   # shell out to rd for this.
 306                   subprocess.check_call([
 307                       'rd', '/s', '/q', os.path.normpath(dst)], shell=True)
 308                 else:
 309                   shutil.rmtree(dst)
 310               break
 311             except OSError as error:
 312               print('Failed rmtree with %s, retrying' % error)
 313               time.sleep(5)
 314           else:
 315             print('Gave up.')
 316             raise OSError('rm: ' + str(error))
 317
 318   except OSError as error:
 319     print(error)
 320
 321   return 0
 322
 323
 324 def MakeZipPath(os_path, isdir, iswindows):
 325   """Changes a path into zipfile format.
 326
 327   # doctest doesn't seem to honor r'' strings, so the backslashes need to be
 328   # escaped.
 329   >>> MakeZipPath(r'C:\\users\\foobar\\blah', False, True)
 330   'users/foobar/blah'
 331   >>> MakeZipPath('/tmp/tmpfoobar/something', False, False)
 332   'tmp/tmpfoobar/something'
 333   >>> MakeZipPath('./somefile.txt', False, False)
 334   'somefile.txt'
 335   >>> MakeZipPath('somedir', True, False)
 336   'somedir/'
 337   >>> MakeZipPath('../dir/filename.txt', False, False)
 338   '../dir/filename.txt'
 339   >>> MakeZipPath('dir/../filename.txt', False, False)
 340   'filename.txt'
 341   """
 342   zip_path = os_path
 343   if iswindows:
 344     import ntpath
 345     # zipfile paths are always posix-style. They also have the drive
 346     # letter and leading slashes removed.
 347     zip_path = ntpath.splitdrive(os_path)[1].replace('\\', '/')
 348   if zip_path.startswith('/'):
 349     zip_path = zip_path[1:]
 350   zip_path = posixpath.normpath(zip_path)
 351   # zipfile also always appends a slash to a directory name.
 352   if isdir:
 353     zip_path += '/'
 354   return zip_path
 355
 356
 357 def OSMakeZipPath(os_path):
 358   return MakeZipPath(os_path, os.path.isdir(os_path), sys.platform == 'win32')
 359
 360
 361 def Zip(args):
 362   """A Unix style zip.
 363
 364   Compresses the listed files.
 365   """
 366   parser = argparse.ArgumentParser(description=Zip.__doc__)
 367   parser.add_argument(
 368       '-r', dest='recursive', action='store_true',
 369       default=False,
 370       help='recurse into directories')
 371   parser.add_argument(
 372       '-q', dest='quiet', action='store_true',
 373       default=False,
 374       help='quiet operation')
 375   parser.add_argument('zipfile')
 376   parser.add_argument('filenames', nargs='+')
 377   options = parser.parse_args(args)
 378
 379   src_files = []
 380   for filename in options.filenames:
 381     globbed_src_args = glob.glob(filename)
 382     if not globbed_src_args:
 383       if not options.quiet:
 384         print('zip warning: name not matched: %s' % filename)
 385
 386     for src_file in globbed_src_args:
 387       src_file = os.path.normpath(src_file)
 388       src_files.append(src_file)
 389       if options.recursive and os.path.isdir(src_file):
 390         for root, dirs, files in os.walk(src_file):
 391           for dirname in dirs:
 392             src_files.append(os.path.join(root, dirname))
 393           for filename in files:
 394             src_files.append(os.path.join(root, filename))
 395
 396   # zip_data represents a list of the data to be written or appended to the
 397   # zip_stream. It is a list of tuples:
 398   #   (OS file path, zip path/zip file info, and file data)
 399   # In all cases one of the |os path| or the |file data| will be None.
 400   # |os path| is None when there is no OS file to write to the archive (i.e.
 401   # the file data already existed in the archive). |file data| is None when the
 402   # file is new (never existed in the archive) or being updated.
 403   zip_data = []
 404   new_files_to_add = [OSMakeZipPath(src_file) for src_file in src_files]
 405   zip_path_to_os_path_dict = dict((new_files_to_add[i], src_files[i])
 406                                   for i in range(len(src_files)))
 407   write_mode = 'a'
 408   if os.path.exists(options.zipfile):
 409     with zipfile.ZipFile(options.zipfile, 'r') as zip_stream:
 410       try:
 411         files_to_update = set(new_files_to_add).intersection(
 412             set(zip_stream.namelist()))
 413         if files_to_update:
 414           # As far as I can tell, there is no way to update a zip entry using
 415           # zipfile; the best you can do is rewrite the archive.
 416           # Iterate through the zipfile to maintain file order.
 417           write_mode = 'w'
 418           for zip_path in zip_stream.namelist():
 419             if zip_path in files_to_update:
 420               os_path = zip_path_to_os_path_dict[zip_path]
 421               zip_data.append((os_path, zip_path, None))
 422               new_files_to_add.remove(zip_path)
 423             else:
 424               file_bytes = zip_stream.read(zip_path)
 425               file_info = zip_stream.getinfo(zip_path)
 426               zip_data.append((None, file_info, file_bytes))
 427       except IOError:
 428         pass
 429
 430   for zip_path in new_files_to_add:
 431     zip_data.append((zip_path_to_os_path_dict[zip_path], zip_path, None))
 432
 433   if not zip_data:
 434     print('zip error: Nothing to do! (%s)' % options.zipfile)
 435     return 1
 436
 437   with zipfile.ZipFile(options.zipfile, write_mode,
 438                        zipfile.ZIP_DEFLATED) as zip_stream:
 439     for os_path, file_info_or_zip_path, file_bytes in zip_data:
 440       if isinstance(file_info_or_zip_path, zipfile.ZipInfo):
 441         zip_path = file_info_or_zip_path.filename
 442       else:
 443         zip_path = file_info_or_zip_path
 444
 445       if os_path:
 446         st = os.stat(os_path)
 447         if stat.S_ISDIR(st.st_mode):
 448           # Python 2.6 on the buildbots doesn't support writing directories to
 449           # zip files. This was resolved in a later version of Python 2.6.
 450           # We'll work around it by writing an empty file with the correct
 451           # path. (This is basically what later versions do anyway.)
 452           zip_info = zipfile.ZipInfo()
 453           zip_info.filename = zip_path
 454           zip_info.date_time = time.localtime(st.st_mtime)[0:6]
 455           zip_info.compress_type = zip_stream.compression
 456           zip_info.flag_bits = 0x00
 457           zip_info.external_attr = (st[0] & 0xFFFF) << 16L
 458           zip_info.CRC = 0
 459           zip_info.compress_size = 0
 460           zip_info.file_size = 0
 461           zip_stream.writestr(zip_info, '')
 462         else:
 463           zip_stream.write(os_path, zip_path)
 464       else:
 465         zip_stream.writestr(file_info_or_zip_path, file_bytes)
 466
 467       if not options.quiet:
 468         if zip_path in new_files_to_add:
 469           operation = 'adding'
 470         else:
 471           operation = 'updating'
 472         zip_info = zip_stream.getinfo(zip_path)
 473         if (zip_info.compress_type == zipfile.ZIP_STORED or
 474             zip_info.file_size == 0):
 475           print('  %s: %s (stored 0%%)' % (operation, zip_path))
 476         elif zip_info.compress_type == zipfile.ZIP_DEFLATED:
 477           print('  %s: %s (deflated %d%%)' % (operation, zip_path,
 478               100 - zip_info.compress_size * 100 / zip_info.file_size))
 479
 480   return 0
 481
 482
 483 def FindExeInPath(filename):
 484   env_path = os.environ.get('PATH', '')
 485   paths = env_path.split(os.pathsep)
 486
 487   def IsExecutableFile(path):
 488     return os.path.isfile(path) and os.access(path, os.X_OK)
 489
 490   if os.path.sep in filename:
 491     if IsExecutableFile(filename):
 492       return filename
 493
 494   for path in paths:
 495     filepath = os.path.join(path, filename)
 496     if IsExecutableFile(filepath):
 497       return os.path.abspath(os.path.join(path, filename))
 498
 499
 500 def Which(args):
 501   """A Unix style which.
 502
 503   Looks for all arguments in the PATH environment variable, and prints their
 504   path if they are executable files.
 505
 506   Note: If you pass an argument with a path to which, it will just test if it
 507   is executable, not if it is in the path.
 508   """
 509   parser = argparse.ArgumentParser(description=Which.__doc__)
 510   parser.add_argument('files', nargs='+')
 511   options = parser.parse_args(args)
 512
 513   retval = 0
 514   for filename in options.files:
 515     fullname = FindExeInPath(filename)
 516     if fullname:
 517       print(fullname)
 518     else:
 519       retval = 1
 520
 521   return retval
 522
 523
 524 FuncMap = {
 525   'cp': Copy,
 526   'mkdir': Mkdir,
 527   'mv': Move,
 528   'rm': Remove,
 529   'zip': Zip,
 530   'which': Which,
 531 }
 532
 533
 534 def main(args):
 535   if not args:
 536     print('No command specified')
 537     print('Available commands: %s' % ' '.join(FuncMap))
 538     return 1
 539   func_name = args[0]
 540   func = FuncMap.get(func_name)
 541   if not func:
 542     print('Do not recognize command: %s' % func_name)
 543     print('Available commands: %s' % ' '.join(FuncMap))
 544     return 1
 545   try:
 546     return func(args[1:])
 547   except KeyboardInterrupt:
 548     print('%s: interrupted' % func_name)
 549     return 1
 550
 551 if __name__ == '__main__':
 552   sys.exit(main(sys.argv[1:]))