tools/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 Before running this script, you should first verify that you are authenticated
  10 for SVN. You can do this by running:
  11   $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
  12 You may need to get your SVN password from https://chromium-access.appspot.com/.
  13
  14 Usage:
  15   $ cd /tmp
  16   $ /path/to/update_reference_build.py VERSION  # e.g. 37.0.2062.94
  17   $ cd reference_builds/reference_builds
  18   $ gcl change
  19   $ gcl upload <change>
  20   $ gcl commit <change>
  21 """
  22
  23 import logging
  24 import optparse
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import time
  30 import urllib
  31 import urllib2
  32 import zipfile
  33
  34
  35 # Google storage location (no public web URL's), example:
  36 # gs://chrome-unsigned/desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
  37 CHROME_GS_URL_FMT = ('gs://chrome-unsigned/desktop-*/%s/%s/%s')
  38
  39
  40 class BuildUpdater(object):
  41   _CHROME_PLATFORM_FILES_MAP = {
  42       'Win': [
  43           'chrome-win.zip',
  44       ],
  45       'Mac': [
  46           'chrome-mac.zip',
  47       ],
  48       'Linux': [
  49           'chrome-precise32.zip',
  50       ],
  51       'Linux_x64': [
  52           'chrome-precise64.zip',
  53       ],
  54   }
  55
  56   # Map of platform names to gs:// Chrome build names.
  57   _BUILD_PLATFORM_MAP = {
  58       'Linux': 'precise32',
  59       'Linux_x64': 'precise64',
  60       'Win': 'win',
  61       'Mac': 'mac',
  62   }
  63
  64   _PLATFORM_DEST_MAP = {
  65       'Linux': 'chrome_linux',
  66       'Linux_x64': 'chrome_linux64',
  67       'Win': 'chrome_win',
  68       'Mac': 'chrome_mac',
  69   }
  70
  71   def __init__(self, version, options):
  72     self._version = version
  73     self._platforms = options.platforms.split(',')
  74
  75   @staticmethod
  76   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
  77     """Executes a subprocess and returns its exit code and output.
  78
  79     Args:
  80       args: A string or a sequence of program arguments.
  81       cwd: If not None, the subprocess's current directory will be changed to
  82         |cwd| before it's executed.
  83       shell: Whether to execute args as a shell command.
  84
  85     Returns:
  86       The tuple (exit code, output).
  87     """
  88     logging.info(str(args) + ' ' + (cwd or ''))
  89     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
  90                          stderr=subprocess.PIPE, shell=shell)
  91     stdout, stderr = p.communicate()
  92     exit_code = p.returncode
  93     if stderr:
  94       logging.critical(stderr)
  95     logging.info(stdout)
  96     return (exit_code, stdout)
  97
  98   def _GetBuildUrl(self, platform, version, filename):
  99     """Returns the URL for fetching one file.
 100
 101     Args:
 102       platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|.
 103       version: A Chrome version number, e.g. 30.0.1600.1.
 104       filename: Name of the file to fetch.
 105
 106     Returns:
 107       The URL for fetching a file. This may be a GS or HTTP URL.
 108     """
 109     return CHROME_GS_URL_FMT % (
 110         version, self._BUILD_PLATFORM_MAP[platform], filename)
 111
 112   def _FindBuildVersion(self, platform, version, filename):
 113     """Searches for a version where a filename can be found.
 114
 115     Args:
 116       platform: Platform name.
 117       version: A Chrome version number, e.g. 30.0.1600.1.
 118       filename: Filename to look for.
 119
 120     Returns:
 121       A version where the file could be found, or None.
 122     """
 123     # TODO(shadi): Iterate over official versions to find a valid one.
 124     return (version
 125             if self._DoesBuildExist(platform, version, filename) else None)
 126
 127   def _DoesBuildExist(self, platform, version, filename):
 128     """Checks whether a file can be found for the given Chrome version.
 129
 130     Args:
 131       platform: Platform name.
 132       version: Chrome version number, e.g. 30.0.1600.1.
 133       filename: Filename to look for.
 134
 135     Returns:
 136       True if the file could be found, False otherwise.
 137     """
 138     url = self._GetBuildUrl(platform, version, filename)
 139     return self._DoesGSFileExist(url)
 140
 141   def _DoesGSFileExist(self, gs_file_name):
 142     """Returns True if the GS file can be found, False otherwise."""
 143     exit_code = BuildUpdater._GetCmdStatusAndOutput(
 144         ['gsutil', 'ls', gs_file_name])[0]
 145     return not exit_code
 146
 147   def _GetPlatformFiles(self, platform):
 148     """Returns a list of filenames to fetch for the given platform."""
 149     return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
 150
 151   def _DownloadBuilds(self):
 152     for platform in self._platforms:
 153       for filename in self._GetPlatformFiles(platform):
 154         output = os.path.join('dl', platform,
 155                               '%s_%s_%s' % (platform,
 156                                             self._version,
 157                                             filename))
 158         if os.path.exists(output):
 159           logging.info('%s alread exists, skipping download', output)
 160           continue
 161         version = self._FindBuildVersion(platform, self._version, filename)
 162         if not version:
 163           logging.critical('Failed to find %s build for r%s\n', platform,
 164                            self._version)
 165           sys.exit(1)
 166         dirname = os.path.dirname(output)
 167         if dirname and not os.path.exists(dirname):
 168           os.makedirs(dirname)
 169         url = self._GetBuildUrl(platform, version, filename)
 170         self._DownloadFile(url, output)
 171
 172   def _DownloadFile(self, url, output):
 173     logging.info('Downloading %s, saving to %s', url, output)
 174     BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
 175
 176   def _FetchSvnRepos(self):
 177     if not os.path.exists('reference_builds'):
 178       os.makedirs('reference_builds')
 179     BuildUpdater._GetCmdStatusAndOutput(
 180         ['gclient', 'config',
 181          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 182         'reference_builds')
 183     BuildUpdater._GetCmdStatusAndOutput(
 184         ['gclient', 'sync'], 'reference_builds')
 185
 186   def _UnzipFile(self, dl_file, dest_dir):
 187     """Unzips a file if it is a zip file.
 188
 189     Args:
 190       dl_file: The downloaded file to unzip.
 191       dest_dir: The destination directory to unzip to.
 192
 193     Returns:
 194       True if the file was unzipped. False if it wasn't a zip file.
 195     """
 196     if not zipfile.is_zipfile(dl_file):
 197       return False
 198     logging.info('Opening %s', dl_file)
 199     with zipfile.ZipFile(dl_file, 'r') as z:
 200       for content in z.namelist():
 201         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 202         # Create dest parent dir if it does not exist.
 203         if not os.path.isdir(os.path.dirname(dest)):
 204           os.makedirs(os.path.dirname(dest))
 205         # If dest is just a dir listing, do nothing.
 206         if not os.path.basename(dest):
 207           continue
 208         if not os.path.isdir(os.path.dirname(dest)):
 209           os.makedirs(os.path.dirname(dest))
 210         with z.open(content) as unzipped_content:
 211           logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
 212           with file(dest, 'wb') as dest_file:
 213             dest_file.write(unzipped_content.read())
 214           permissions = z.getinfo(content).external_attr >> 16
 215           if permissions:
 216             os.chmod(dest, permissions)
 217     return True
 218
 219   def _ClearDir(self, dir):
 220     """Clears all files in |dir| except for hidden files and folders."""
 221     for root, dirs, files in os.walk(dir):
 222       # Skip hidden files and folders (like .svn and .git).
 223       files = [f for f in files if f[0] != '.']
 224       dirs[:] = [d for d in dirs if d[0] != '.']
 225
 226       for f in files:
 227         os.remove(os.path.join(root, f))
 228
 229   def _ExtractBuilds(self):
 230     for platform in self._platforms:
 231       if os.path.exists('tmp_unzip'):
 232         os.path.unlink('tmp_unzip')
 233       dest_dir = os.path.join('reference_builds', 'reference_builds',
 234                               BuildUpdater._PLATFORM_DEST_MAP[platform])
 235       self._ClearDir(dest_dir)
 236       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 237         for dl_file in dl_files:
 238           dl_file = os.path.join(root, dl_file)
 239           if not self._UnzipFile(dl_file, dest_dir):
 240             logging.info('Copying %s to %s', dl_file, dest_dir)
 241             shutil.copy(dl_file, dest_dir)
 242
 243   def _SvnAddAndRemove(self):
 244     svn_dir = os.path.join('reference_builds', 'reference_builds')
 245     # List all changes without ignoring any files.
 246     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
 247                                                svn_dir)[1]
 248     for line in stat.splitlines():
 249       action, filename = line.split(None, 1)
 250       # Add new and ignored files.
 251       if action == '?' or action == 'I':
 252         BuildUpdater._GetCmdStatusAndOutput(
 253             ['svn', 'add', filename], svn_dir)
 254       elif action == '!':
 255         BuildUpdater._GetCmdStatusAndOutput(
 256             ['svn', 'delete', filename], svn_dir)
 257       filepath = os.path.join(svn_dir, filename)
 258       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 259         BuildUpdater._GetCmdStatusAndOutput(
 260             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 261
 262   def DownloadAndUpdateBuilds(self):
 263     self._DownloadBuilds()
 264     self._FetchSvnRepos()
 265     self._ExtractBuilds()
 266     self._SvnAddAndRemove()
 267
 268
 269 def ParseOptions(argv):
 270   parser = optparse.OptionParser()
 271   parser.set_usage('Usage: %prog VERSION [-p PLATFORMS]')
 272   parser.add_option('-p', dest='platforms',
 273                     default='Win,Mac,Linux,Linux_x64',
 274                     help='Comma separated list of platforms to download '
 275                          '(as defined by the chromium builders).')
 276
 277   options, args = parser.parse_args(argv)
 278   if len(args) != 2:
 279     parser.print_help()
 280     sys.exit(1)
 281   version = args[1]
 282
 283   return version, options
 284
 285
 286 def main(argv):
 287   logging.getLogger().setLevel(logging.DEBUG)
 288   version, options = ParseOptions(argv)
 289   b = BuildUpdater(version, options)
 290   b.DownloadAndUpdateBuilds()
 291   logging.info('Successfully updated reference builds. Move to '
 292                'reference_builds/reference_builds and make a change with gcl.')
 293
 294 if __name__ == '__main__':
 295   sys.exit(main(sys.argv))