tools/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 Use -r option to update a Chromium reference build, or -b option for Chrome
  10 official builds.
  11
  12 Usage:
  13   $ cd /tmp
  14   $ /path/to/update_reference_build.py -r <revision>
  15   $ cd reference_builds/reference_builds
  16   $ gcl change
  17   $ gcl upload <change>
  18   $ gcl commit <change>
  19 """
  20
  21 import logging
  22 import optparse
  23 import os
  24 import shutil
  25 import subprocess
  26 import sys
  27 import time
  28 import urllib
  29 import urllib2
  30 import zipfile
  31
  32 # Example chromium build location:
  33 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip
  34 CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
  35                     'chromium-browser-snapshots/%s/%s/%s')
  36
  37 # Chrome official build storage
  38 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds
  39
  40 # Internal Google archive of official Chrome builds, example:
  41 # https://goto.google.com/chrome_official_builds/
  42 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip
  43 CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/'
  44                            'official_builds/%s/%s/%s')
  45
  46 # Google storage location (no public web URL's), example:
  47 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip
  48 CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s')
  49
  50
  51 class BuildUpdater(object):
  52   _PLATFORM_FILES_MAP = {
  53       'Win': [
  54           'chrome-win32.zip',
  55           'chrome-win32-syms.zip',
  56       ],
  57       'Mac': [
  58           'chrome-mac.zip',
  59       ],
  60       'Linux': [
  61           'chrome-linux.zip',
  62       ],
  63       'Linux_x64': [
  64           'chrome-linux.zip',
  65       ],
  66   }
  67
  68   _CHROME_PLATFORM_FILES_MAP = {
  69       'Win': [
  70           'chrome-win32.zip',
  71           'chrome-win32-syms.zip',
  72       ],
  73       'Mac': [
  74           'chrome-mac.zip',
  75       ],
  76       'Linux': [
  77           'chrome-precise32bit.zip',
  78       ],
  79       'Linux_x64': [
  80           'chrome-precise64bit.zip',
  81       ],
  82   }
  83
  84   # Map of platform names to gs:// Chrome build names.
  85   _BUILD_PLATFORM_MAP = {
  86       'Linux': 'precise32bit',
  87       'Linux_x64': 'precise64bit',
  88       'Win': 'win',
  89       'Mac': 'mac',
  90   }
  91
  92   _PLATFORM_DEST_MAP = {
  93       'Linux': 'chrome_linux',
  94       'Linux_x64': 'chrome_linux64',
  95       'Win': 'chrome_win',
  96       'Mac': 'chrome_mac',
  97   }
  98
  99   def __init__(self, options):
 100     self._platforms = options.platforms.split(',')
 101     self._revision = options.build_number or int(options.revision)
 102     self._use_build_number = bool(options.build_number)
 103     self._use_gs = options.use_gs
 104
 105   @staticmethod
 106   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
 107     """Executes a subprocess and returns its exit code and output.
 108
 109     Args:
 110       args: A string or a sequence of program arguments.
 111       cwd: If not None, the subprocess's current directory will be changed to
 112         |cwd| before it's executed.
 113       shell: Whether to execute args as a shell command.
 114
 115     Returns:
 116       The tuple (exit code, output).
 117     """
 118     logging.info(str(args) + ' ' + (cwd or ''))
 119     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
 120                          stderr=subprocess.PIPE, shell=shell)
 121     stdout, stderr = p.communicate()
 122     exit_code = p.returncode
 123     if stderr:
 124       logging.critical(stderr)
 125     logging.info(stdout)
 126     return (exit_code, stdout)
 127
 128   def _GetBuildUrl(self, platform, revision, filename):
 129     if self._use_build_number:
 130       # Chrome Google storage bucket.
 131       if self._use_gs:
 132         release = revision[:revision.find('.')]
 133         return (CHROME_GS_URL_FMT % (
 134             release,
 135             revision,
 136             self._BUILD_PLATFORM_MAP[platform],
 137             filename))
 138       # Chrome internal archive.
 139       return (CHROME_INTERNAL_URL_FMT % (
 140           revision,
 141           self._BUILD_PLATFORM_MAP[platform],
 142           filename))
 143     # Chromium archive.
 144     return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename)
 145
 146   def _FindBuildRevision(self, platform, revision, filename):
 147     # TODO(shadi): Iterate over build numbers to find a valid one.
 148     if self._use_build_number:
 149       return (revision
 150               if self._DoesBuildExist(platform, revision, filename) else None)
 151
 152     MAX_REVISIONS_PER_BUILD = 100
 153     for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
 154       if self._DoesBuildExist(platform, revision_guess, filename):
 155         return revision_guess
 156       else:
 157         time.sleep(.1)
 158     return None
 159
 160   def _DoesBuildExist(self, platform, build_number, filename):
 161     url = self._GetBuildUrl(platform, build_number, filename)
 162     if self._use_gs:
 163       return self._DoesGSFileExist(url)
 164
 165     r = urllib2.Request(url)
 166     r.get_method = lambda: 'HEAD'
 167     try:
 168       urllib2.urlopen(r)
 169       return True
 170     except urllib2.HTTPError, err:
 171       if err.code == 404:
 172         return False
 173
 174   def _DoesGSFileExist(self, gs_file_name):
 175     exit_code = BuildUpdater._GetCmdStatusAndOutput(
 176         ['gsutil', 'ls', gs_file_name])[0]
 177     return not exit_code
 178
 179   def _GetPlatformFiles(self, platform):
 180     if self._use_build_number:
 181       return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
 182     return BuildUpdater._PLATFORM_FILES_MAP[platform]
 183
 184   def _DownloadBuilds(self):
 185     for platform in self._platforms:
 186       for f in self._GetPlatformFiles(platform):
 187         output = os.path.join('dl', platform,
 188                               '%s_%s_%s' % (platform, self._revision, f))
 189         if os.path.exists(output):
 190           logging.info('%s alread exists, skipping download', output)
 191           continue
 192         build_revision = self._FindBuildRevision(platform, self._revision, f)
 193         if not build_revision:
 194           logging.critical('Failed to find %s build for r%s\n', platform,
 195                            self._revision)
 196           sys.exit(1)
 197         dirname = os.path.dirname(output)
 198         if dirname and not os.path.exists(dirname):
 199           os.makedirs(dirname)
 200         url = self._GetBuildUrl(platform, build_revision, f)
 201         self._DownloadFile(url, output)
 202
 203   def _DownloadFile(self, url, output):
 204     logging.info('Downloading %s, saving to %s', url, output)
 205     if self._use_build_number and self._use_gs:
 206       BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
 207     else:
 208       r = urllib2.urlopen(url)
 209       with file(output, 'wb') as f:
 210         f.write(r.read())
 211
 212   def _FetchSvnRepos(self):
 213     if not os.path.exists('reference_builds'):
 214       os.makedirs('reference_builds')
 215     BuildUpdater._GetCmdStatusAndOutput(
 216         ['gclient', 'config',
 217          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 218         'reference_builds')
 219     BuildUpdater._GetCmdStatusAndOutput(
 220         ['gclient', 'sync'], 'reference_builds')
 221
 222   def _UnzipFile(self, dl_file, dest_dir):
 223     if not zipfile.is_zipfile(dl_file):
 224       return False
 225     logging.info('Opening %s', dl_file)
 226     with zipfile.ZipFile(dl_file, 'r') as z:
 227       for content in z.namelist():
 228         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 229         # Create dest parent dir if it does not exist.
 230         if not os.path.isdir(os.path.dirname(dest)):
 231           os.makedirs(os.path.dirname(dest))
 232         # If dest is just a dir listing, do nothing.
 233         if not os.path.basename(dest):
 234           continue
 235         if not os.path.isdir(os.path.dirname(dest)):
 236           os.makedirs(os.path.dirname(dest))
 237         with z.open(content) as unzipped_content:
 238           logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
 239           with file(dest, 'wb') as dest_file:
 240             dest_file.write(unzipped_content.read())
 241           permissions = z.getinfo(content).external_attr >> 16
 242           if permissions:
 243             os.chmod(dest, permissions)
 244     return True
 245
 246   def _ClearDir(self, dir):
 247     """Clears all files in |dir| except for hidden files and folders."""
 248     for root, dirs, files in os.walk(dir):
 249       # Skip hidden files and folders (like .svn and .git).
 250       files = [f for f in files if f[0] != '.']
 251       dirs[:] = [d for d in dirs if d[0] != '.']
 252
 253       for f in files:
 254         os.remove(os.path.join(root, f))
 255
 256   def _ExtractBuilds(self):
 257     for platform in self._platforms:
 258       if os.path.exists('tmp_unzip'):
 259         os.path.unlink('tmp_unzip')
 260       dest_dir = os.path.join('reference_builds', 'reference_builds',
 261                               BuildUpdater._PLATFORM_DEST_MAP[platform])
 262       self._ClearDir(dest_dir)
 263       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 264         for dl_file in dl_files:
 265           dl_file = os.path.join(root, dl_file)
 266           if not self._UnzipFile(dl_file, dest_dir):
 267             logging.info('Copying %s to %s', dl_file, dest_dir)
 268             shutil.copy(dl_file, dest_dir)
 269
 270   def _SvnAddAndRemove(self):
 271     svn_dir = os.path.join('reference_builds', 'reference_builds')
 272     # List all changes without ignoring any files.
 273     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
 274                                                svn_dir)[1]
 275     for line in stat.splitlines():
 276       action, filename = line.split(None, 1)
 277       # Add new and ignored files.
 278       if action == '?' or action == 'I':
 279         BuildUpdater._GetCmdStatusAndOutput(
 280             ['svn', 'add', filename], svn_dir)
 281       elif action == '!':
 282         BuildUpdater._GetCmdStatusAndOutput(
 283             ['svn', 'delete', filename], svn_dir)
 284       filepath = os.path.join(svn_dir, filename)
 285       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 286         BuildUpdater._GetCmdStatusAndOutput(
 287             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 288
 289   def DownloadAndUpdateBuilds(self):
 290     self._DownloadBuilds()
 291     self._FetchSvnRepos()
 292     self._ExtractBuilds()
 293     self._SvnAddAndRemove()
 294
 295
 296 def ParseOptions(argv):
 297   parser = optparse.OptionParser()
 298   usage = 'usage: %prog <options>'
 299   parser.set_usage(usage)
 300   parser.add_option('-b', dest='build_number',
 301                     help='Chrome official build number to pick up.')
 302   parser.add_option('--gs', dest='use_gs', action='store_true', default=False,
 303                     help='Use Google storage for official builds. Used with -b '
 304                          'option. Default is false (i.e. use internal storage.')
 305   parser.add_option('-p', dest='platforms',
 306                     default='Win,Mac,Linux,Linux_x64',
 307                     help='Comma separated list of platforms to download '
 308                          '(as defined by the chromium builders).')
 309   parser.add_option('-r', dest='revision',
 310                     help='Revision to pick up.')
 311
 312   (options, _) = parser.parse_args(argv)
 313   if not options.revision and not options.build_number:
 314     logging.critical('Must specify either -r or -b.\n')
 315     sys.exit(1)
 316   if options.revision and options.build_number:
 317     logging.critical('Must specify either -r or -b but not both.\n')
 318     sys.exit(1)
 319   if options.use_gs and not options.build_number:
 320     logging.critical('Can only use --gs with -b option.\n')
 321     sys.exit(1)
 322
 323   return options
 324
 325
 326 def main(argv):
 327   logging.getLogger().setLevel(logging.DEBUG)
 328   options = ParseOptions(argv)
 329   b = BuildUpdater(options)
 330   b.DownloadAndUpdateBuilds()
 331   logging.info('Successfully updated reference builds. Move to '
 332                'reference_builds/reference_builds and make a change with gcl.')
 333
 334 if __name__ == '__main__':
 335   sys.exit(main(sys.argv))