build/get_syzygy_binaries.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """A utility script for downloading versioned Syzygy binaries."""
   7
   8 import cStringIO
   9 import hashlib
  10 import json
  11 import logging
  12 import optparse
  13 import os
  14 import re
  15 import shutil
  16 import subprocess
  17 import urllib2
  18 import zipfile
  19
  20
  21 _LOGGER = logging.getLogger(os.path.basename(__file__))
  22
  23 # The URL where official builds are archived.
  24 _SYZYGY_ARCHIVE_URL = ('http://syzygy-archive.commondatastorage.googleapis.com/'
  25     'builds/official/%(revision)s')
  26
  27 # A JSON file containing the state of the download directory. If this file and
  28 # directory state do not agree, then the binaries will be downloaded and
  29 # installed again.
  30 _STATE = '.state'
  31
  32 # This matches an integer (an SVN revision number) or a SHA1 value (a GIT hash).
  33 # The archive exclusively uses lowercase GIT hashes.
  34 _REVISION_RE = re.compile('^(?:\d+|[a-f0-9]{40})$')
  35
  36 # This matches an MD5 hash.
  37 _MD5_RE = re.compile('^[a-f0-9]{32}$')
  38
  39 # List of reources to be downloaded and installed. These are tuples with the
  40 # following format:
  41 # (basename, logging name, relative installation path, extraction filter)
  42 _RESOURCES = [
  43   ('benchmark.zip', 'benchmark', '', None),
  44   ('binaries.zip', 'binaries', 'exe', None),
  45   ('symbols.zip', 'symbols', 'symbols',
  46       lambda x: x.filename.endswith('.dll.pdb')),
  47   ('include.zip', 'include', 'include', None),
  48   ('lib.zip', 'library', 'lib', None)]
  49
  50
  51 def _Shell(*cmd, **kw):
  52   """Runs |cmd|, returns the results from Popen(cmd).communicate()."""
  53   _LOGGER.debug('Executing %s.', cmd)
  54   prog = subprocess.Popen(cmd, shell=True, **kw)
  55
  56   stdout, stderr = prog.communicate()
  57   if prog.returncode != 0:
  58     raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
  59   return (stdout, stderr)
  60
  61
  62 def _LoadState(output_dir):
  63   """Loads the contents of the state file for a given |output_dir|, returning
  64   None if it doesn't exist.
  65   """
  66   path = os.path.join(output_dir, _STATE)
  67   if not os.path.exists(path):
  68     _LOGGER.debug('No state file found.')
  69     return None
  70   with open(path, 'rb') as f:
  71     _LOGGER.debug('Reading state file: %s', path)
  72     try:
  73       return json.load(f)
  74     except ValueError:
  75       _LOGGER.debug('Invalid state file.')
  76       return None
  77
  78
  79 def _SaveState(output_dir, state, dry_run=False):
  80   """Saves the |state| dictionary to the given |output_dir| as a JSON file."""
  81   path = os.path.join(output_dir, _STATE)
  82   _LOGGER.debug('Writing state file: %s', path)
  83   if dry_run:
  84     return
  85   with open(path, 'wb') as f:
  86     f.write(json.dumps(state, sort_keys=True, indent=2))
  87
  88
  89 def _Md5(path):
  90   """Returns the MD5 hash of the file at |path|, which must exist."""
  91   return hashlib.md5(open(path, 'rb').read()).hexdigest()
  92
  93
  94 def _StateIsValid(state):
  95   """Returns true if the given state structure is valid."""
  96   if not isinstance(state, dict):
  97     _LOGGER.debug('State must be a dict.')
  98     return False
  99   r = state.get('revision', None)
 100   if not isinstance(r, basestring) or not _REVISION_RE.match(r):
 101     _LOGGER.debug('State contains an invalid revision.')
 102     return False
 103   c = state.get('contents', None)
 104   if not isinstance(c, dict):
 105     _LOGGER.debug('State must contain a contents dict.')
 106     return False
 107   for (relpath, md5) in c.iteritems():
 108     if not isinstance(relpath, basestring) or len(relpath) == 0:
 109       _LOGGER.debug('State contents dict contains an invalid path.')
 110       return False
 111     if not isinstance(md5, basestring) or not _MD5_RE.match(md5):
 112       _LOGGER.debug('State contents dict contains an invalid MD5 digest.')
 113       return False
 114   return True
 115
 116
 117 def _BuildActualState(stored, revision, output_dir):
 118   """Builds the actual state using the provided |stored| state as a template.
 119   Only examines files listed in the stored state, causing the script to ignore
 120   files that have been added to the directories locally. |stored| must be a
 121   valid state dictionary.
 122   """
 123   contents = {}
 124   state = { 'revision': revision, 'contents': contents }
 125   for relpath, md5 in stored['contents'].iteritems():
 126     abspath = os.path.abspath(os.path.join(output_dir, relpath))
 127     if os.path.isfile(abspath):
 128       m = _Md5(abspath)
 129       contents[relpath] = m
 130
 131   return state
 132
 133
 134 def _StatesAreConsistent(stored, actual):
 135   """Validates whether two state dictionaries are consistent. Both must be valid
 136   state dictionaries. Additional entries in |actual| are ignored.
 137   """
 138   if stored['revision'] != actual['revision']:
 139     _LOGGER.debug('Mismatched revision number.')
 140     return False
 141   cont_stored = stored['contents']
 142   cont_actual = actual['contents']
 143   for relpath, md5 in cont_stored.iteritems():
 144     if relpath not in cont_actual:
 145       _LOGGER.debug('Missing content: %s', relpath)
 146       return False
 147     if md5 != cont_actual[relpath]:
 148       _LOGGER.debug('Modified content: %s', relpath)
 149       return False
 150   return True
 151
 152
 153 def _GetCurrentState(revision, output_dir):
 154   """Loads the current state and checks to see if it is consistent. Returns
 155   a tuple (state, bool). The returned state will always be valid, even if an
 156   invalid state is present on disk.
 157   """
 158   stored = _LoadState(output_dir)
 159   if not _StateIsValid(stored):
 160     _LOGGER.debug('State is invalid.')
 161     # Return a valid but empty state.
 162     return ({'revision': '0', 'contents': {}}, False)
 163   actual = _BuildActualState(stored, revision, output_dir)
 164   if not _StatesAreConsistent(stored, actual):
 165     return (stored, False)
 166   return (stored, True)
 167
 168
 169 def _DirIsEmpty(path):
 170   """Returns true if the given directory is empty, false otherwise."""
 171   for root, dirs, files in os.walk(path):
 172     return not dirs and not files
 173
 174
 175 def _CleanState(output_dir, state, dry_run=False):
 176   """Cleans up files/directories in |output_dir| that are referenced by
 177   the given |state|. Raises an error if there are local changes. Returns a
 178   dictionary of files that were deleted.
 179   """
 180   _LOGGER.debug('Deleting files from previous installation.')
 181   deleted = {}
 182
 183   # Generate a list of files to delete, relative to |output_dir|.
 184   contents = state['contents']
 185   files = sorted(contents.keys())
 186
 187   # Try to delete the files. Keep track of directories to delete as well.
 188   dirs = {}
 189   for relpath in files:
 190     fullpath = os.path.join(output_dir, relpath)
 191     fulldir = os.path.dirname(fullpath)
 192     dirs[fulldir] = True
 193     if os.path.exists(fullpath):
 194       # If somehow the file has become a directory complain about it.
 195       if os.path.isdir(fullpath):
 196         raise Exception('Directory exists where file expected: %s' % fullpath)
 197
 198       # Double check that the file doesn't have local changes. If it does
 199       # then refuse to delete it.
 200       if relpath in contents:
 201         stored_md5 = contents[relpath]
 202         actual_md5 = _Md5(fullpath)
 203         if actual_md5 != stored_md5:
 204           raise Exception('File has local changes: %s' % fullpath)
 205
 206       # The file is unchanged so it can safely be deleted.
 207       _LOGGER.debug('Deleting file "%s".', fullpath)
 208       deleted[relpath] = True
 209       if not dry_run:
 210         os.unlink(fullpath)
 211
 212   # Sort directories from longest name to shortest. This lets us remove empty
 213   # directories from the most nested paths first.
 214   dirs = sorted(dirs.keys(), key=lambda x: len(x), reverse=True)
 215   for p in dirs:
 216     if os.path.exists(p) and _DirIsEmpty(p):
 217       _LOGGER.debug('Deleting empty directory "%s".', p)
 218       if not dry_run:
 219         shutil.rmtree(p, False)
 220
 221   return deleted
 222
 223
 224 def _Download(url):
 225   """Downloads the given URL and returns the contents as a string."""
 226   response = urllib2.urlopen(url)
 227   if response.code != 200:
 228     raise RuntimeError('Failed to download "%s".' % url)
 229   return response.read()
 230
 231
 232 def _InstallBinaries(options, deleted={}):
 233   """Installs Syzygy binaries. This assumes that the output directory has
 234   already been cleaned, as it will refuse to overwrite existing files."""
 235   contents = {}
 236   state = { 'revision': options.revision, 'contents': contents }
 237   archive_url = _SYZYGY_ARCHIVE_URL % { 'revision': options.revision }
 238   for (base, name, subdir, filt) in _RESOURCES:
 239     # Create the output directory if it doesn't exist.
 240     fulldir = os.path.join(options.output_dir, subdir)
 241     if os.path.isfile(fulldir):
 242       raise Exception('File exists where a directory needs to be created: %s' %
 243                       fulldir)
 244     if not os.path.exists(fulldir):
 245       _LOGGER.debug('Creating directory: %s', fulldir)
 246       if not options.dry_run:
 247         os.makedirs(fulldir)
 248
 249     # Download the archive.
 250     url = archive_url + '/' + base
 251     _LOGGER.debug('Retrieving %s archive at "%s".', name, url)
 252     data = _Download(url)
 253
 254     _LOGGER.debug('Unzipping %s archive.', name)
 255     archive = zipfile.ZipFile(cStringIO.StringIO(data))
 256     for entry in archive.infolist():
 257       if not filt or filt(entry):
 258         fullpath = os.path.normpath(os.path.join(fulldir, entry.filename))
 259         relpath = os.path.relpath(fullpath, options.output_dir)
 260         if os.path.exists(fullpath):
 261           # If in a dry-run take into account the fact that the file *would*
 262           # have been deleted.
 263           if options.dry_run and relpath in deleted:
 264             pass
 265           else:
 266             raise Exception('Path already exists: %s' % fullpath)
 267
 268         # Extract the file and update the state dictionary.
 269         _LOGGER.debug('Extracting "%s".', fullpath)
 270         if not options.dry_run:
 271           archive.extract(entry.filename, fulldir)
 272           md5 = _Md5(fullpath)
 273           contents[relpath] = md5
 274
 275   return state
 276
 277
 278 def _ParseCommandLine():
 279   """Parses the command-line and returns an options structure."""
 280   option_parser = optparse.OptionParser()
 281   option_parser.add_option('--dry-run', action='store_true', default=False,
 282       help='If true then will simply list actions that would be performed.')
 283   option_parser.add_option('--force', action='store_true', default=False,
 284       help='Force an installation even if the binaries are up to date.')
 285   option_parser.add_option('--output-dir', type='string',
 286       help='The path where the binaries will be replaced. Existing binaries '
 287            'will only be overwritten if not up to date.')
 288   option_parser.add_option('--overwrite', action='store_true', default=False,
 289       help='If specified then the installation will happily delete and rewrite '
 290            'the entire output directory, blasting any local changes.')
 291   option_parser.add_option('--revision', type='string',
 292       help='The SVN revision or GIT hash associated with the required version.')
 293   option_parser.add_option('--revision-file', type='string',
 294       help='A text file containing an SVN revision or GIT hash.')
 295   option_parser.add_option('--verbose', dest='log_level', action='store_const',
 296       default=logging.INFO, const=logging.DEBUG,
 297       help='Enables verbose logging.')
 298   option_parser.add_option('--quiet', dest='log_level', action='store_const',
 299       default=logging.INFO, const=logging.ERROR,
 300       help='Disables all output except for errors.')
 301   options, args = option_parser.parse_args()
 302   if args:
 303     option_parser.error('Unexpected arguments: %s' % args)
 304   if not options.output_dir:
 305     option_parser.error('Must specify --output-dir.')
 306   if not options.revision and not options.revision_file:
 307     option_parser.error('Must specify one of --revision or --revision-file.')
 308   if options.revision and options.revision_file:
 309     option_parser.error('Must not specify both --revision and --revision-file.')
 310
 311   # Configure logging.
 312   logging.basicConfig(level=options.log_level)
 313
 314   # If a revision file has been specified then read it.
 315   if options.revision_file:
 316     options.revision = open(options.revision_file, 'rb').read().strip()
 317     _LOGGER.debug('Parsed revision "%s" from file "%s".',
 318                  options.revision, options.revision_file)
 319
 320   # Ensure that the specified SVN revision or GIT hash is valid.
 321   if not _REVISION_RE.match(options.revision):
 322     option_parser.error('Must specify a valid SVN or GIT revision.')
 323
 324   # This just makes output prettier to read.
 325   options.output_dir = os.path.normpath(options.output_dir)
 326
 327   return options
 328
 329
 330 def main():
 331   options = _ParseCommandLine()
 332
 333   if options.dry_run:
 334     _LOGGER.debug('Performing a dry-run.')
 335
 336   # Load the current installation state, and validate it against the
 337   # requested installation.
 338   state, is_consistent = _GetCurrentState(options.revision, options.output_dir)
 339
 340   # Decide whether or not an install is necessary.
 341   if options.force:
 342     _LOGGER.debug('Forcing reinstall of binaries.')
 343   elif is_consistent:
 344     # Avoid doing any work if the contents of the directory are consistent.
 345     _LOGGER.debug('State unchanged, no reinstall necessary.')
 346     return
 347
 348   # Under normal logging this is the only only message that will be reported.
 349   _LOGGER.info('Installing revision %s Syzygy binaries.',
 350                options.revision[0:12])
 351
 352   # Clean up the old state to begin with.
 353   deleted = []
 354   if options.overwrite:
 355     if os.path.exists(options.output_dir):
 356       # If overwrite was specified then take a heavy-handed approach.
 357       _LOGGER.debug('Deleting entire installation directory.')
 358       if not options.dry_run:
 359         shutil.rmtree(options.output_dir, False)
 360   else:
 361     # Otherwise only delete things that the previous installation put in place,
 362     # and take care to preserve any local changes.
 363     deleted = _CleanState(options.output_dir, state, options.dry_run)
 364
 365   # Install the new binaries. In a dry-run this will actually download the
 366   # archives, but it won't write anything to disk.
 367   state = _InstallBinaries(options, deleted)
 368
 369   # Build and save the state for the directory.
 370   _SaveState(options.output_dir, state, options.dry_run)
 371
 372
 373 if __name__ == '__main__':
 374   main()