tools/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 Usage:
  10   $ cd /tmp
  11   $ /path/to/update_reference_build.py -r <revision>
  12   $ cd reference_builds/reference_builds
  13   $ gcl change
  14   $ gcl upload <change>
  15   $ gcl commit <change>
  16 """
  17
  18 import errno
  19 import logging
  20 import optparse
  21 import os
  22 import shutil
  23 import subprocess
  24 import sys
  25 import time
  26 import urllib
  27 import urllib2
  28 import zipfile
  29
  30
  31 class BuildUpdater(object):
  32   _PLATFORM_FILES_MAP = {
  33     'Win': [
  34         'chrome-win32.zip',
  35         'chrome-win32-syms.zip',
  36         'chrome-win32.test/_pyautolib.pyd',
  37         'chrome-win32.test/pyautolib.py',
  38     ],
  39     'Mac': [
  40       'chrome-mac.zip',
  41       'chrome-mac.test/_pyautolib.so',
  42       'chrome-mac.test/pyautolib.py',
  43     ],
  44     'Linux': [
  45         'chrome-linux.zip',
  46     ],
  47     'Linux_x64': [
  48         'chrome-linux.zip',
  49     ],
  50   }
  51
  52   _PLATFORM_DEST_MAP = {
  53     'Linux': 'chrome_linux',
  54     'Linux_x64': 'chrome_linux64',
  55     'Win': 'chrome_win',
  56     'Mac': 'chrome_mac',
  57    }
  58
  59   def __init__(self, options):
  60     self._platforms = options.platforms.split(',')
  61     self._revision = int(options.revision)
  62
  63   @staticmethod
  64   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
  65     """Executes a subprocess and returns its exit code and output.
  66
  67     Args:
  68       args: A string or a sequence of program arguments.
  69       cwd: If not None, the subprocess's current directory will be changed to
  70         |cwd| before it's executed.
  71       shell: Whether to execute args as a shell command.
  72
  73     Returns:
  74       The tuple (exit code, output).
  75     """
  76     logging.info(str(args) + ' ' + (cwd or ''))
  77     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
  78                          stderr=subprocess.PIPE, shell=shell)
  79     stdout, stderr = p.communicate()
  80     exit_code = p.returncode
  81     if stderr:
  82       logging.critical(stderr)
  83     logging.info(stdout)
  84     return (exit_code, stdout)
  85
  86   def _GetBuildUrl(self, platform, revision, filename):
  87     URL_FMT = ('http://commondatastorage.googleapis.com/'
  88                'chromium-browser-snapshots/%s/%s/%s')
  89     return URL_FMT % (urllib.quote_plus(platform), revision, filename)
  90
  91   def _FindBuildRevision(self, platform, revision, filename):
  92     MAX_REVISIONS_PER_BUILD = 100
  93     for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
  94       r = urllib2.Request(self._GetBuildUrl(platform, revision_guess, filename))
  95       r.get_method = lambda: 'HEAD'
  96       try:
  97         response = urllib2.urlopen(r)
  98         return revision_guess
  99       except urllib2.HTTPError, err:
 100         if err.code == 404:
 101           time.sleep(.1)
 102           continue
 103     return None
 104
 105   def _DownloadBuilds(self):
 106     for platform in self._platforms:
 107       for f in BuildUpdater._PLATFORM_FILES_MAP[platform]:
 108         output = os.path.join('dl', platform,
 109                               '%s_%s_%s' % (platform, self._revision, f))
 110         if os.path.exists(output):
 111           logging.info('%s alread exists, skipping download' % output)
 112           continue
 113         build_revision = self._FindBuildRevision(platform, self._revision, f)
 114         if not build_revision:
 115           logging.critical('Failed to find %s build for r%s\n' % (
 116               platform, self._revision))
 117           sys.exit(1)
 118         dirname = os.path.dirname(output)
 119         if dirname and not os.path.exists(dirname):
 120           os.makedirs(dirname)
 121         url = self._GetBuildUrl(platform, build_revision, f)
 122         logging.info('Downloading %s, saving to %s' % (url, output))
 123         r = urllib2.urlopen(url)
 124         with file(output, 'wb') as f:
 125           f.write(r.read())
 126
 127   def _FetchSvnRepos(self):
 128     if not os.path.exists('reference_builds'):
 129       os.makedirs('reference_builds')
 130     BuildUpdater._GetCmdStatusAndOutput(
 131         ['gclient', 'config',
 132          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 133         'reference_builds')
 134     BuildUpdater._GetCmdStatusAndOutput(
 135         ['gclient', 'sync'], 'reference_builds')
 136
 137   def _UnzipFile(self, dl_file, dest_dir):
 138     if not zipfile.is_zipfile(dl_file):
 139       return False
 140     logging.info('Opening %s' % dl_file)
 141     with zipfile.ZipFile(dl_file, 'r') as z:
 142       for content in z.namelist():
 143         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 144         if not os.path.basename(dest):
 145           if not os.path.isdir(dest):
 146             os.makedirs(dest)
 147           continue
 148         with z.open(content) as unzipped_content:
 149           logging.info('Extracting %s to %s (%s)' % (content, dest, dl_file))
 150           with file(dest, 'wb') as dest_file:
 151             dest_file.write(unzipped_content.read())
 152           permissions = z.getinfo(content).external_attr >> 16
 153           if permissions:
 154             os.chmod(dest, permissions)
 155     return True
 156
 157   def _ClearDir(self, dir):
 158     """Clears all files in |dir| except for hidden files and folders."""
 159     for root, dirs, files in os.walk(dir):
 160       # Skip hidden files and folders (like .svn and .git).
 161       files = [f for f in files if f[0] != '.']
 162       dirs[:] = [d for d in dirs if d[0] != '.']
 163
 164       for f in files:
 165         os.remove(os.path.join(root, f))
 166
 167   def _ExtractBuilds(self):
 168     for platform in self._platforms:
 169       if os.path.exists('tmp_unzip'):
 170         os.path.unlink('tmp_unzip')
 171       dest_dir = os.path.join('reference_builds', 'reference_builds',
 172                               BuildUpdater._PLATFORM_DEST_MAP[platform])
 173       self._ClearDir(dest_dir)
 174       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 175         for dl_file in dl_files:
 176           dl_file = os.path.join(root, dl_file)
 177           if not self._UnzipFile(dl_file, dest_dir):
 178             logging.info('Copying %s to %s' % (dl_file, dest_dir))
 179             shutil.copy(dl_file, dest_dir)
 180
 181   def _SvnAddAndRemove(self):
 182     svn_dir = os.path.join('reference_builds', 'reference_builds')
 183     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat'], svn_dir)[1]
 184     for line in stat.splitlines():
 185       action, filename = line.split(None, 1)
 186       if action == '?':
 187         BuildUpdater._GetCmdStatusAndOutput(
 188             ['svn', 'add', filename], svn_dir)
 189       elif action == '!':
 190         BuildUpdater._GetCmdStatusAndOutput(
 191             ['svn', 'delete', filename], svn_dir)
 192       filepath = os.path.join(svn_dir, filename)
 193       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 194         BuildUpdater._GetCmdStatusAndOutput(
 195             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 196
 197   def DownloadAndUpdateBuilds(self):
 198     self._DownloadBuilds()
 199     self._FetchSvnRepos()
 200     self._ExtractBuilds()
 201     self._SvnAddAndRemove()
 202
 203
 204 def ParseOptions(argv):
 205   parser = optparse.OptionParser()
 206   usage = 'usage: %prog <options>'
 207   parser.set_usage(usage)
 208   parser.add_option('-r', dest='revision',
 209                     help='Revision to pickup')
 210   parser.add_option('-p', dest='platforms',
 211                     default='Win,Mac,Linux,Linux_x64',
 212                     help='Comma separated list of platforms to download '
 213                          '(as defined by the chromium builders).')
 214   (options, _) = parser.parse_args(argv)
 215   if not options.revision:
 216     logging.critical('Must specify -r\n')
 217     sys.exit(1)
 218
 219   return options
 220
 221
 222 def main(argv):
 223   logging.getLogger().setLevel(logging.DEBUG)
 224   options = ParseOptions(argv)
 225   b = BuildUpdater(options)
 226   b.DownloadAndUpdateBuilds()
 227   logging.info('Successfully updated reference builds. Move to '
 228                'reference_builds/reference_builds and make a change with gcl.')
 229
 230 if __name__ == '__main__':
 231   sys.exit(main(sys.argv))