tools/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 Use -r option to update a Chromium reference build, or -b option for Chrome
  10 official builds.
  11
  12 Usage:
  13   $ cd /tmp
  14   $ /path/to/update_reference_build.py -r <revision>
  15   $ cd reference_builds/reference_builds
  16   $ gcl change
  17   $ gcl upload <change>
  18   $ gcl commit <change>
  19 """
  20
  21 import logging
  22 import optparse
  23 import os
  24 import shutil
  25 import subprocess
  26 import sys
  27 import time
  28 import urllib
  29 import urllib2
  30 import zipfile
  31
  32 # Example chromium build location:
  33 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip
  34 CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
  35                     'chromium-browser-snapshots/%s/%s/%s')
  36
  37 # Chrome official build storage
  38 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds
  39
  40 # Internal Google archive of official Chrome builds, example:
  41 # https://goto.google.com/chrome_official_builds/
  42 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip
  43 CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/'
  44                            'official_builds/%s/%s/%s')
  45
  46 # Google storage location (no public web URL's), example:
  47 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip
  48 CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s')
  49
  50
  51 class BuildUpdater(object):
  52   _PLATFORM_FILES_MAP = {
  53       'Win': [
  54           'chrome-win32.zip',
  55           'chrome-win32-syms.zip',
  56       ],
  57       'Mac': [
  58           'chrome-mac.zip',
  59       ],
  60       'Linux': [
  61           'chrome-linux.zip',
  62       ],
  63       'Linux_x64': [
  64           'chrome-linux.zip',
  65       ],
  66   }
  67
  68   _CHROME_PLATFORM_FILES_MAP = {
  69       'Win': [
  70           'chrome-win32.zip',
  71           'chrome-win32-syms.zip',
  72       ],
  73       'Mac': [
  74           'chrome-mac.zip',
  75       ],
  76       'Linux': [
  77           'chrome-precise32bit.zip',
  78       ],
  79       'Linux_x64': [
  80           'chrome-precise64bit.zip',
  81       ],
  82   }
  83
  84   # Map of platform names to gs:// Chrome build names.
  85   _BUILD_PLATFORM_MAP = {
  86       'Linux': 'precise32bit',
  87       'Linux_x64': 'precise64bit',
  88       'Win': 'win',
  89       'Mac': 'mac',
  90   }
  91
  92   _PLATFORM_DEST_MAP = {
  93       'Linux': 'chrome_linux',
  94       'Linux_x64': 'chrome_linux64',
  95       'Win': 'chrome_win',
  96       'Mac': 'chrome_mac',
  97   }
  98
  99   def __init__(self, options):
 100     self._platforms = options.platforms.split(',')
 101     self._revision = options.build_number or int(options.revision)
 102     self._use_build_number = bool(options.build_number)
 103     self._use_gs = options.use_gs
 104
 105   @staticmethod
 106   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
 107     """Executes a subprocess and returns its exit code and output.
 108
 109     Args:
 110       args: A string or a sequence of program arguments.
 111       cwd: If not None, the subprocess's current directory will be changed to
 112         |cwd| before it's executed.
 113       shell: Whether to execute args as a shell command.
 114
 115     Returns:
 116       The tuple (exit code, output).
 117     """
 118     logging.info(str(args) + ' ' + (cwd or ''))
 119     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
 120                          stderr=subprocess.PIPE, shell=shell)
 121     stdout, stderr = p.communicate()
 122     exit_code = p.returncode
 123     if stderr:
 124       logging.critical(stderr)
 125     logging.info(stdout)
 126     return (exit_code, stdout)
 127
 128   def _GetBuildUrl(self, platform, revision, filename):
 129     if self._use_build_number:
 130       # Chrome Google storage bucket.
 131       if self._use_gs:
 132         release = revision[:revision.find('.')]
 133         return (CHROME_GS_URL_FMT % (
 134             release,
 135             revision,
 136             self._BUILD_PLATFORM_MAP[platform],
 137             filename))
 138       # Chrome internal archive.
 139       return (CHROME_INTERNAL_URL_FMT % (
 140           revision,
 141           self._BUILD_PLATFORM_MAP[platform],
 142           filename))
 143     # Chromium archive.
 144     return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename)
 145
 146   def _FindBuildRevision(self, platform, revision, filename):
 147     # TODO(shadi): Iterate over build numbers to find a valid one.
 148     if self._use_build_number:
 149       return (revision
 150               if self._DoesBuildExist(platform, revision, filename) else None)
 151
 152     MAX_REVISIONS_PER_BUILD = 100
 153     for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
 154       if self._DoesBuildExist(platform, revision_guess, filename):
 155         return revision_guess
 156       else:
 157         time.sleep(.1)
 158     return None
 159
 160   def _DoesBuildExist(self, platform, build_number, filename):
 161     url = self._GetBuildUrl(platform, build_number, filename)
 162     if self._use_gs:
 163       return self._DoesGSFileExist(url)
 164
 165     r = urllib2.Request(url)
 166     r.get_method = lambda: 'HEAD'
 167     try:
 168       urllib2.urlopen(r)
 169       return True
 170     except urllib2.HTTPError, err:
 171       if err.code == 404:
 172         return False
 173
 174   def _DoesGSFileExist(self, gs_file_name):
 175     exit_code = BuildUpdater._GetCmdStatusAndOutput(
 176         ['gsutil', 'ls', gs_file_name])[0]
 177     return not exit_code
 178
 179   def _GetPlatformFiles(self, platform):
 180     if self._use_build_number:
 181       return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
 182     return BuildUpdater._PLATFORM_FILES_MAP[platform]
 183
 184   def _DownloadBuilds(self):
 185     for platform in self._platforms:
 186       for f in self._GetPlatformFiles(platform):
 187         output = os.path.join('dl', platform,
 188                               '%s_%s_%s' % (platform, self._revision, f))
 189         if os.path.exists(output):
 190           logging.info('%s alread exists, skipping download', output)
 191           continue
 192         build_revision = self._FindBuildRevision(platform, self._revision, f)
 193         if not build_revision:
 194           logging.critical('Failed to find %s build for r%s\n', platform,
 195                            self._revision)
 196           sys.exit(1)
 197         dirname = os.path.dirname(output)
 198         if dirname and not os.path.exists(dirname):
 199           os.makedirs(dirname)
 200         url = self._GetBuildUrl(platform, build_revision, f)
 201         self._DownloadFile(url, output)
 202
 203   def _DownloadFile(self, url, output):
 204     logging.info('Downloading %s, saving to %s', url, output)
 205     if self._use_build_number and self._use_gs:
 206       BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
 207     else:
 208       r = urllib2.urlopen(url)
 209       with file(output, 'wb') as f:
 210         f.write(r.read())
 211
 212   def _FetchSvnRepos(self):
 213     if not os.path.exists('reference_builds'):
 214       os.makedirs('reference_builds')
 215     BuildUpdater._GetCmdStatusAndOutput(
 216         ['gclient', 'config',
 217          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 218         'reference_builds')
 219     BuildUpdater._GetCmdStatusAndOutput(
 220         ['gclient', 'sync'], 'reference_builds')
 221
 222   def _UnzipFile(self, dl_file, dest_dir):
 223     if not zipfile.is_zipfile(dl_file):
 224       return False
 225     logging.info('Opening %s', dl_file)
 226     with zipfile.ZipFile(dl_file, 'r') as z:
 227       for content in z.namelist():
 228         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 229         # Create dest parent dir if it does not exist.
 230         if not os.path.isdir(os.path.dirname(dest)):
 231           os.makedirs(os.path.dirname(dest))
 232         # If dest is just a dir listing, do nothing.
 233         if not os.path.basename(dest):
 234           continue
 235         with z.open(content) as unzipped_content:
 236           logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
 237           with file(dest, 'wb') as dest_file:
 238             dest_file.write(unzipped_content.read())
 239           permissions = z.getinfo(content).external_attr >> 16
 240           if permissions:
 241             os.chmod(dest, permissions)
 242     return True
 243
 244   def _ClearDir(self, dir):
 245     """Clears all files in |dir| except for hidden files and folders."""
 246     for root, dirs, files in os.walk(dir):
 247       # Skip hidden files and folders (like .svn and .git).
 248       files = [f for f in files if f[0] != '.']
 249       dirs[:] = [d for d in dirs if d[0] != '.']
 250
 251       for f in files:
 252         os.remove(os.path.join(root, f))
 253
 254   def _ExtractBuilds(self):
 255     for platform in self._platforms:
 256       if os.path.exists('tmp_unzip'):
 257         os.path.unlink('tmp_unzip')
 258       dest_dir = os.path.join('reference_builds', 'reference_builds',
 259                               BuildUpdater._PLATFORM_DEST_MAP[platform])
 260       self._ClearDir(dest_dir)
 261       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 262         for dl_file in dl_files:
 263           dl_file = os.path.join(root, dl_file)
 264           if not self._UnzipFile(dl_file, dest_dir):
 265             logging.info('Copying %s to %s', dl_file, dest_dir)
 266             shutil.copy(dl_file, dest_dir)
 267
 268   def _SvnAddAndRemove(self):
 269     svn_dir = os.path.join('reference_builds', 'reference_builds')
 270     # List all changes without ignoring any files.
 271     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
 272                                                svn_dir)[1]
 273     for line in stat.splitlines():
 274       action, filename = line.split(None, 1)
 275       # Add new and ignored files.
 276       if action == '?' or action == 'I':
 277         BuildUpdater._GetCmdStatusAndOutput(
 278             ['svn', 'add', filename], svn_dir)
 279       elif action == '!':
 280         BuildUpdater._GetCmdStatusAndOutput(
 281             ['svn', 'delete', filename], svn_dir)
 282       filepath = os.path.join(svn_dir, filename)
 283       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 284         BuildUpdater._GetCmdStatusAndOutput(
 285             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 286
 287   def DownloadAndUpdateBuilds(self):
 288     self._DownloadBuilds()
 289     self._FetchSvnRepos()
 290     self._ExtractBuilds()
 291     self._SvnAddAndRemove()
 292
 293
 294 def ParseOptions(argv):
 295   parser = optparse.OptionParser()
 296   usage = 'usage: %prog <options>'
 297   parser.set_usage(usage)
 298   parser.add_option('-b', dest='build_number',
 299                     help='Chrome official build number to pick up.')
 300   parser.add_option('--gs', dest='use_gs', action='store_true', default=False,
 301                     help='Use Google storage for official builds. Used with -b '
 302                          'option. Default is false (i.e. use internal storage.')
 303   parser.add_option('-p', dest='platforms',
 304                     default='Win,Mac,Linux,Linux_x64',
 305                     help='Comma separated list of platforms to download '
 306                          '(as defined by the chromium builders).')
 307   parser.add_option('-r', dest='revision',
 308                     help='Revision to pick up.')
 309
 310   (options, _) = parser.parse_args(argv)
 311   if not options.revision and not options.build_number:
 312     logging.critical('Must specify either -r or -b.\n')
 313     sys.exit(1)
 314   if options.revision and options.build_number:
 315     logging.critical('Must specify either -r or -b but not both.\n')
 316     sys.exit(1)
 317   if options.use_gs and not options.build_number:
 318     logging.critical('Can only use --gs with -b option.\n')
 319     sys.exit(1)
 320
 321   return options
 322
 323
 324 def main(argv):
 325   logging.getLogger().setLevel(logging.DEBUG)
 326   options = ParseOptions(argv)
 327   b = BuildUpdater(options)
 328   b.DownloadAndUpdateBuilds()
 329   logging.info('Successfully updated reference builds. Move to '
 330                'reference_builds/reference_builds and make a change with gcl.')
 331
 332 if __name__ == '__main__':
 333   sys.exit(main(sys.argv))