From 97ec9ac8703dc10eaef5e60984bb3ecdfad344bd Mon Sep 17 00:00:00 2001 From: dpranke Date: Thu, 16 Jul 2015 17:45:40 -0700 Subject: [PATCH] Add a GN auto-roller script. This CL adds a script in //tools/gn/bin/roll_gn.py that, given no arguments, will post three CLs and three tryjobs in order to: * build a new GN binary at tip-of-tree * update the buildtools repo to the newly built binaries * and then update DEPS to point to the leatest buildtools The script must be run on Linux by someone with commit access to both src/ and buildtools/. One can also run the steps independently for testing purposes. It can only be run in a clean chromium checkout; it should error out in most cases if something bad happens, but the error checking isn't yet foolproof. R=brettw@chromium.org, scottmg@chromium.org, thakis@chromium.org Review URL: https://codereview.chromium.org/1230293003 Cr-Commit-Position: refs/heads/master@{#339191} --- tools/gn/bin/roll_gn.py | 384 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 384 insertions(+) create mode 100644 tools/gn/bin/roll_gn.py diff --git a/tools/gn/bin/roll_gn.py b/tools/gn/bin/roll_gn.py new file mode 100644 index 000000000000..6691ecc13070 --- /dev/null +++ b/tools/gn/bin/roll_gn.py @@ -0,0 +1,384 @@ +# Copyright 2014 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""An auto-roller for GN binaries into Chromium. + +This script is used to update the GN binaries that a Chromium +checkout uses. In order to update the binaries, one must follow +four steps in order: + +1. Trigger try jobs to build a new GN binary at tip-of-tree and upload + the newly-built binaries into the right Google CloudStorage bucket. +2. Wait for the try jobs to complete. +3. Update the buildtools repo with the .sha1 hashes of the newly built + binaries. +4. Update Chromium's DEPS file to the new version of the buildtools repo. + +The script has four commands that correspond to the four steps above: +'build', 'wait', 'roll_buildtools', and 'roll_deps'. + +The script has a fifth command, 'roll', that runs the four in order. + +If given no arguments, the script will run the 'roll' command. + +It can only be run on linux in a clean Chromium checkout; it should +error out in most cases if something bad happens, but the error checking +isn't yet foolproof. + +""" + +from __future__ import print_function + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import time +import urllib2 + +depot_tools_path = None +for p in os.environ['PATH'].split(os.pathsep): + if (p.rstrip(os.sep).endswith('depot_tools') and + os.path.isfile(os.path.join(p, 'gclient.py'))): + depot_tools_path = p + +assert depot_tools_path +if not depot_tools_path in sys.path: + sys.path.insert(0, depot_tools_path) + +third_party_path = os.path.join(depot_tools_path, 'third_party') +if not third_party_path in sys.path: + sys.path.insert(0, third_party_path) + +import upload + + +CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git' + +CODE_REVIEW_SERVER = 'https://codereview.chromium.org' + + +class GNRoller(object): + def __init__(self): + self.chromium_src_dir = None + self.buildtools_dir = None + self.old_gn_commitish = None + self.new_gn_commitish = None + self.old_gn_version = None + self.new_gn_version = None + self.reviewer = 'dpranke@chromium.org' + if os.getenv('USER') == 'dpranke': + self.reviewer = 'brettw@chromium.org' + + def Roll(self): + parser = argparse.ArgumentParser() + parser.usage = __doc__ + parser.add_argument('command', nargs='?', default='roll', + help='build|roll|roll_buildtools|roll_deps|wait' + ' (%(default)s is the default)') + + args = parser.parse_args() + command = args.command + ret = self.SetUp() + if not ret and command in ('roll', 'build'): + ret = self.TriggerBuild() + if not ret and command in ('roll', 'wait'): + ret = self.WaitForBuildToFinish() + if not ret and command in ('roll', 'roll_buildtools'): + ret = self.RollBuildtools() + if not ret and command in ('roll', 'roll_deps'): + ret = self.RollDEPS() + + return ret + + def SetUp(self): + if sys.platform != 'linux2': + print('roll_gn is only tested and working on Linux for now.') + return 1 + + ret, out, _ = self.Call('git config --get remote.origin.url') + origin = out.strip() + if ret or origin != CHROMIUM_REPO: + print('Not in a Chromium repo? git config --get remote.origin.url ' + 'returned %d: %s' % (ret, origin)) + return 1 + + ret, _, _ = self.Call('git diff -q') + if ret: + print("Checkout is dirty, exiting") + return 1 + + _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd()) + self.chromium_src_dir = out.strip() + self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools') + + self.new_gn_commitish, self.new_gn_version = self.GetNewVersions() + + _, out, _ = self.Call('gn --version') + self.old_gn_version = out.strip() + + _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version) + self.old_gn_commitish = out.strip() + return 0 + + def GetNewVersions(self): + _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position') + commit_msg = out.splitlines() + first_line = commit_msg[0] + new_gn_commitish = first_line.split()[1] + + last_line = commit_msg[-1] + new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line) + + return new_gn_commitish, new_gn_version + + def TriggerBuild(self): + ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version) + if ret: + print('Failed to create a new branch for build_gn_%s' % + self.new_gn_version) + return 1 + + self.MakeDummyDepsChange() + + ret, out, err = self.Call('git commit -a -m "Build gn at %s"' % + self.new_gn_version) + if ret: + print('git commit failed: %s' % out + err) + return 1 + + print('Uploading CL to build GN at {#%s} - %s' % + (self.new_gn_version, self.new_gn_commitish)) + ret, out, err = self.Call('git cl upload -f') + if ret: + print('git-cl upload failed: %s' % out + err) + return 1 + + print('Starting try jobs') + self.Call('git-cl try -b linux_chromium_gn_upload ' + '-b mac_chromium_gn_upload ' + '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish) + + return 0 + + def MakeDummyDepsChange(self): + with open('DEPS') as fp: + deps_content = fp.read() + new_deps = deps_content.replace("'buildtools_revision':", + "'buildtools_revision': ") + + with open('DEPS', 'w') as fp: + fp.write(new_deps) + + def WaitForBuildToFinish(self): + print('Checking build') + results = self.CheckBuild() + while any(r['state'] == 'pending' for r in results.values()): + print() + print('Sleeping for 30 seconds') + time.sleep(30) + print('Checking build') + results = self.CheckBuild() + return 0 if all(r['state'] == 'success' for r in results.values()) else 1 + + def CheckBuild(self): + _, out, _ = self.Call('git-cl issue') + + issue = int(out.split()[2]) + + _, out, _ = self.Call('git config user.email') + email = '' + rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email) + try: + props = json.loads(rpc_server.Send('/api/%d' % issue)) + except Exception as _e: + raise + + patchset = int(props['patchsets'][-1]) + + try: + patchset_data = json.loads(rpc_server.Send('/api/%d/%d' % + (issue, patchset))) + except Exception as _e: + raise + + TRY_JOB_RESULT_STATES = ('success', 'warnings', 'failure', 'skipped', + 'exception', 'retry', 'pending') + try_job_results = patchset_data['try_job_results'] + if not try_job_results: + print('No try jobs found on most recent patchset') + return 1 + + results = {} + for job in try_job_results: + builder = job['builder'] + if builder == 'linux_chromium_gn_upload': + platform = 'linux64' + elif builder == 'mac_chromium_gn_upload': + platform = 'mac' + elif builder == 'win8_chromium_gn_upload': + platform = 'win' + else: + print('Unexpected builder: %s') + continue + + state = TRY_JOB_RESULT_STATES[int(job['result'])] + url_str = ' %s' % job['url'] + build = url_str.split('/')[-1] + + sha1 = '-' + results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str}) + + if state == 'success': + jsurl = url_str.replace('/builders/', '/json/builders/') + fp = urllib2.urlopen(jsurl) + js = json.loads(fp.read()) + fp.close() + for step in js['steps']: + if step['name'] == 'gn sha1': + sha1 = step['text'][1] + + if results[platform]['build'] < build: + results[platform]['build'] = build + results[platform]['sha1'] = sha1 + results[platform]['state'] = state + results[platform]['url'] = url_str + + for platform, r in results.items(): + print(platform) + print(' sha1: %s' % r['sha1']) + print(' state: %s' % r['state']) + print(' build: %s' % r['build']) + print(' url: %s' % r['url']) + print() + + return results + + def RollBuildtools(self): + results = self.CheckBuild() + if not all(r['state'] == 'success' for r in results.values()): + print("Roll isn't done or didn't succeed, exiting:") + return 1 + + desc = self.GetBuildtoolsDesc() + + self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version, + cwd=self.buildtools_dir) + + for platform in results: + fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1' + path = os.path.join(self.buildtools_dir, platform, fname) + with open(path, 'w') as fp: + fp.write('%s\n' % results[platform]['sha1']) + + desc_file = tempfile.NamedTemporaryFile(delete=False) + try: + desc_file.write(desc) + desc_file.close() + self.Call('git commit -a -F %s' % desc_file.name, + cwd=self.buildtools_dir) + self.Call('git-cl upload -f --send-mail', + cwd=self.buildtools_dir) + finally: + os.remove(desc_file.name) + + self.Call('git cl push', cwd=self.buildtools_dir) + + # Fetch the revision we just committed so that RollDEPS will find it. + self.Call('git cl fetch', cwd=self.buildtools_dir) + + return 0 + + def RollDEPS(self): + _, out, _ = self.Call('git rev-parse origin/master', + cwd=self.buildtools_dir) + new_buildtools_commitish = out.strip() + + new_deps_lines = [] + old_buildtools_commitish = '' + with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp: + for l in fp.readlines(): + m = re.match(".*'buildtools_revision':.*'(.+)',", l) + if m: + old_buildtools_commitish = m.group(1) + new_deps_lines.append(" 'buildtools_revision': '%s'," % + new_buildtools_commitish) + else: + new_deps_lines.append(l) + + if not old_buildtools_commitish: + print('Could not update DEPS properly, exiting') + return 1 + + with open('DEPS', 'w') as fp: + fp.write(''.join(new_deps_lines) + '\n') + + desc = self.GetDEPSRollDesc(old_buildtools_commitish, + new_buildtools_commitish) + desc_file = tempfile.NamedTemporaryFile(delete=False) + try: + desc_file.write(desc) + desc_file.close() + self.Call('git commit -a -F %s' % desc_file.name) + self.Call('git-cl upload -f --send-mail --commit-queue') + finally: + os.remove(desc_file.name) + return 0 + + def GetBuildtoolsDesc(self): + gn_changes = self.GetGNChanges() + return ( + 'Roll gn %s..%s (r%s:%s)\n' + '\n' + '%s' + '\n' + 'TBR=%s\n' % ( + self.old_gn_commitish, + self.new_gn_commitish, + self.old_gn_version, + self.new_gn_version, + gn_changes, + self.reviewer, + )) + + def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish): + gn_changes = self.GetGNChanges() + + return ( + 'Roll DEPS %s..%s\n' + '\n' + ' in order to roll GN %s..%s (r%s:%s)\n' + '\n' + '%s' + '\n' + 'TBR=%s\n' % ( + old_buildtools_commitish, + new_buildtools_commitish, + self.old_gn_commitish, + self.new_gn_commitish, + self.old_gn_version, + self.new_gn_version, + gn_changes, + self.reviewer, + )) + + def GetGNChanges(self): + _, out, _ = self.Call( + "git log --pretty=' %h %s' " + + "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish)) + return out + + def Call(self, cmd, cwd=None): + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, + cwd=(cwd or self.chromium_src_dir)) + out, err = proc.communicate() + return proc.returncode, out, err + + +if __name__ == '__main__': + roller = GNRoller() + sys.exit(roller.Roll()) -- 2.11.4.GIT