tools/roll_angle.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import collections
   8 import logging
   9 import os
  10 import re
  11 import subprocess
  12 import sys
  13 import time
  14
  15
  16 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
  17 SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
  18 import find_depot_tools
  19 find_depot_tools.add_depot_tools_to_path()
  20 import roll_dep_svn
  21 from gclient import GClientKeywords
  22 from third_party import upload
  23
  24 # Avoid depot_tools/third_party/upload.py print verbose messages.
  25 upload.verbosity = 0  # Errors only.
  26
  27 CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git'
  28 CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$')
  29 RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)')
  30 ROLL_BRANCH_NAME = 'special_angle_roll_branch'
  31 TRYJOB_STATUS_SLEEP_SECONDS = 30
  32
  33 # Use a shell for subcommands on Windows to get a PATH search.
  34 USE_SHELL = sys.platform.startswith('win')
  35 ANGLE_PATH = os.path.join('third_party', 'angle')
  36
  37 CommitInfo = collections.namedtuple('CommitInfo', ['git_commit',
  38                                                    'git_repo_url'])
  39 CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server'])
  40
  41 def _PosixPath(path):
  42   """Convert a possibly-Windows path to a posix-style path."""
  43   (_, path) = os.path.splitdrive(path)
  44   return path.replace(os.sep, '/')
  45
  46 def _ParseGitCommitHash(description):
  47   for line in description.splitlines():
  48     if line.startswith('commit '):
  49       return line.split()[1]
  50   logging.error('Failed to parse git commit id from:\n%s\n', description)
  51   sys.exit(-1)
  52   return None
  53
  54
  55 def _ParseDepsFile(filename):
  56   with open(filename, 'rb') as f:
  57     deps_content = f.read()
  58   return _ParseDepsDict(deps_content)
  59
  60
  61 def _ParseDepsDict(deps_content):
  62   local_scope = {}
  63   var = GClientKeywords.VarImpl({}, local_scope)
  64   global_scope = {
  65     'File': GClientKeywords.FileImpl,
  66     'From': GClientKeywords.FromImpl,
  67     'Var': var.Lookup,
  68     'deps_os': {},
  69   }
  70   exec(deps_content, global_scope, local_scope)
  71   return local_scope
  72
  73
  74 def _GenerateCLDescriptionCommand(angle_current, angle_new, bugs):
  75   def GetChangeString(current_hash, new_hash):
  76     return '%s..%s' % (current_hash[0:7], new_hash[0:7]);
  77
  78   def GetChangeLogURL(git_repo_url, change_string):
  79     return '%s/+log/%s' % (git_repo_url, change_string)
  80
  81   def GetBugString(bugs):
  82     bug_str = 'BUG='
  83     for bug in bugs:
  84       bug_str += str(bug) + ','
  85     return bug_str.rstrip(',')
  86
  87   if angle_current.git_commit != angle_new.git_commit:
  88     change_str = GetChangeString(angle_current.git_commit,
  89                                  angle_new.git_commit)
  90     changelog_url = GetChangeLogURL(angle_current.git_repo_url,
  91                                     change_str)
  92
  93   return [
  94     '-m', 'Roll ANGLE ' + change_str,
  95     '-m', '%s' % changelog_url,
  96     '-m', GetBugString(bugs),
  97     '-m', 'TEST=bots',
  98   ]
  99
 100
 101 class AutoRoller(object):
 102   def __init__(self, chromium_src):
 103     self._chromium_src = chromium_src
 104
 105   def _RunCommand(self, command, working_dir=None, ignore_exit_code=False,
 106                   extra_env=None):
 107     """Runs a command and returns the stdout from that command.
 108
 109     If the command fails (exit code != 0), the function will exit the process.
 110     """
 111     working_dir = working_dir or self._chromium_src
 112     logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir)
 113     env = os.environ.copy()
 114     if extra_env:
 115       logging.debug('extra env: %s', extra_env)
 116       env.update(extra_env)
 117     p = subprocess.Popen(command, stdout=subprocess.PIPE,
 118                          stderr=subprocess.PIPE, shell=USE_SHELL, env=env,
 119                          cwd=working_dir, universal_newlines=True)
 120     output = p.stdout.read()
 121     p.wait()
 122     p.stdout.close()
 123     p.stderr.close()
 124
 125     if not ignore_exit_code and p.returncode != 0:
 126       logging.error('Command failed: %s\n%s', str(command), output)
 127       sys.exit(p.returncode)
 128     return output
 129
 130   def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None):
 131     working_dir = os.path.join(self._chromium_src, path_below_src)
 132     self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir)
 133     revision_range = git_hash or 'origin'
 134     ret = self._RunCommand(
 135         ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'],
 136         working_dir=working_dir)
 137     return CommitInfo(_ParseGitCommitHash(ret), git_repo_url)
 138
 139   def _GetDepsCommitInfo(self, deps_dict, path_below_src):
 140     entry = deps_dict['deps'][_PosixPath('src/%s' % path_below_src)]
 141     at_index = entry.find('@')
 142     git_repo_url = entry[:at_index]
 143     git_hash = entry[at_index + 1:]
 144     return self._GetCommitInfo(path_below_src, git_hash, git_repo_url)
 145
 146   def _GetCLInfo(self):
 147     cl_output = self._RunCommand(['git', 'cl', 'issue'])
 148     m = CL_ISSUE_RE.match(cl_output.strip())
 149     if not m:
 150       logging.error('Cannot find any CL info. Output was:\n%s', cl_output)
 151       sys.exit(-1)
 152     issue_number = int(m.group(1))
 153     url = m.group(2)
 154
 155     # Parse the Rietveld host from the URL.
 156     m = RIETVELD_URL_RE.match(url)
 157     if not m:
 158       logging.error('Cannot parse Rietveld host from URL: %s', url)
 159       sys.exit(-1)
 160     rietveld_server = m.group(1)
 161     return CLInfo(issue_number, url, rietveld_server)
 162
 163   def _GetCurrentBranchName(self):
 164     return self._RunCommand(
 165         ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0]
 166
 167   def _IsTreeClean(self):
 168     lines = self._RunCommand(
 169       ['git', 'status', '--porcelain', '-uno']).splitlines()
 170     if len(lines) == 0:
 171       return True
 172
 173     logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines))
 174     return False
 175
 176   def _GetBugList(self, path_below_src, angle_current, angle_new):
 177     working_dir = os.path.join(self._chromium_src, path_below_src)
 178     lines = self._RunCommand(
 179         ['git','log',
 180             '%s..%s' % (angle_current.git_commit, angle_new.git_commit)],
 181         working_dir=working_dir).split('\n')
 182     bugs = set()
 183     for line in lines:
 184       line = line.strip()
 185       bug_prefix = 'BUG='
 186       if line.startswith(bug_prefix):
 187         bugs_strings = line[len(bug_prefix):].split(',')
 188         for bug_string in bugs_strings:
 189           try:
 190             bugs.add(int(bug_string))
 191           except:
 192             # skip this, it may be a project specific bug such as
 193             # "angleproject:X" or an ill-formed BUG= message
 194             pass
 195     return bugs
 196
 197   def _UpdateReadmeFile(self, readme_path, new_revision):
 198     readme = open(os.path.join(self._chromium_src, readme_path), 'r+')
 199     txt = readme.read()
 200     m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE),
 201         ('Revision: %s' % new_revision), txt)
 202     readme.seek(0)
 203     readme.write(m)
 204     readme.truncate()
 205
 206   def PrepareRoll(self, ignore_checks):
 207     # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for
 208     # cross platform compatibility.
 209
 210     if not ignore_checks:
 211       if self._GetCurrentBranchName() != 'master':
 212         logging.error('Please checkout the master branch.')
 213         return -1
 214       if not self._IsTreeClean():
 215         logging.error('Please make sure you don\'t have any modified files.')
 216         return -1
 217
 218     # Always clean up any previous roll.
 219     self.Abort()
 220
 221     logging.debug('Pulling latest changes')
 222     if not ignore_checks:
 223       self._RunCommand(['git', 'pull'])
 224
 225     self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
 226
 227     # Modify Chromium's DEPS file.
 228
 229     # Parse current hashes.
 230     deps_filename = os.path.join(self._chromium_src, 'DEPS')
 231     deps = _ParseDepsFile(deps_filename)
 232     angle_current = self._GetDepsCommitInfo(deps, ANGLE_PATH)
 233
 234     # Find ToT revisions.
 235     angle_latest = self._GetCommitInfo(ANGLE_PATH)
 236
 237     # Make sure the roll script doesn't use windows line endings
 238     self._RunCommand(['git', 'config', 'core.autocrlf', 'true'])
 239
 240     self._UpdateDep(deps_filename, ANGLE_PATH, angle_latest)
 241
 242     if self._IsTreeClean():
 243       logging.debug('Tree is clean - no changes detected.')
 244       self._DeleteRollBranch()
 245     else:
 246       bugs = self._GetBugList(ANGLE_PATH, angle_current, angle_latest)
 247       description = _GenerateCLDescriptionCommand(
 248           angle_current, angle_latest, bugs)
 249       logging.debug('Committing changes locally.')
 250       self._RunCommand(['git', 'add', '--update', '.'])
 251       self._RunCommand(['git', 'commit'] + description)
 252       logging.debug('Uploading changes...')
 253       self._RunCommand(['git', 'cl', 'upload'],
 254                        extra_env={'EDITOR': 'true'})
 255       self._RunCommand(['git', 'cl', 'try'])
 256       cl_info = self._GetCLInfo()
 257       print 'Issue: %d URL: %s' % (cl_info.issue, cl_info.url)
 258
 259     # Checkout master again.
 260     self._RunCommand(['git', 'checkout', 'master'])
 261     print 'Roll branch left as ' + ROLL_BRANCH_NAME
 262     return 0
 263
 264   def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info):
 265     dep_name = _PosixPath(os.path.join('src', dep_relative_to_src))
 266
 267     # roll_dep_svn.py relies on cwd being the Chromium checkout, so let's
 268     # temporarily change the working directory and then change back.
 269     cwd = os.getcwd()
 270     os.chdir(os.path.dirname(deps_filename))
 271     roll_dep_svn.update_deps(deps_filename, dep_relative_to_src, dep_name,
 272                              commit_info.git_commit, '')
 273     os.chdir(cwd)
 274
 275   def _DeleteRollBranch(self):
 276     self._RunCommand(['git', 'checkout', 'master'])
 277     self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 278     logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME)
 279
 280
 281   def _GetBranches(self):
 282     """Returns a tuple of active,branches.
 283
 284     The 'active' is the name of the currently active branch and 'branches' is a
 285     list of all branches.
 286     """
 287     lines = self._RunCommand(['git', 'branch']).split('\n')
 288     branches = []
 289     active = ''
 290     for l in lines:
 291       if '*' in l:
 292         # The assumption is that the first char will always be the '*'.
 293         active = l[1:].strip()
 294         branches.append(active)
 295       else:
 296         b = l.strip()
 297         if b:
 298           branches.append(b)
 299     return (active, branches)
 300
 301   def Abort(self):
 302     active_branch, branches = self._GetBranches()
 303     if active_branch == ROLL_BRANCH_NAME:
 304       active_branch = 'master'
 305     if ROLL_BRANCH_NAME in branches:
 306       print 'Aborting pending roll.'
 307       self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
 308       # Ignore an error here in case an issue wasn't created for some reason.
 309       self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True)
 310       self._RunCommand(['git', 'checkout', active_branch])
 311       self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 312     return 0
 313
 314
 315 def main():
 316   parser = argparse.ArgumentParser(
 317       description='Auto-generates a CL containing an ANGLE roll.')
 318   parser.add_argument('--abort',
 319     help=('Aborts a previously prepared roll. '
 320           'Closes any associated issues and deletes the roll branches'),
 321     action='store_true')
 322   parser.add_argument('--ignore-checks', action='store_true', default=False,
 323       help=('Skips checks for being on the master branch, dirty workspaces and '
 324             'the updating of the checkout. Will still delete and create local '
 325             'Git branches.'))
 326   parser.add_argument('-v', '--verbose', action='store_true', default=False,
 327       help='Be extra verbose in printing of log messages.')
 328   args = parser.parse_args()
 329
 330   if args.verbose:
 331     logging.basicConfig(level=logging.DEBUG)
 332   else:
 333     logging.basicConfig(level=logging.ERROR)
 334
 335   autoroller = AutoRoller(SRC_DIR)
 336   if args.abort:
 337     return autoroller.Abort()
 338   else:
 339     return autoroller.PrepareRoll(args.ignore_checks)
 340
 341 if __name__ == '__main__':
 342   sys.exit(main())