tools/gn/bin/roll_gn.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """An auto-roller for GN binaries into Chromium.
   6
   7 This script is used to update the GN binaries that a Chromium
   8 checkout uses. In order to update the binaries, one must follow
   9 four steps in order:
  10
  11 1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
  12    the newly-built binaries into the right Google CloudStorage bucket.
  13 2. Wait for the try jobs to complete.
  14 3. Update the buildtools repo with the .sha1 hashes of the newly built
  15    binaries.
  16 4. Update Chromium's DEPS file to the new version of the buildtools repo.
  17
  18 The script has four commands that correspond to the four steps above:
  19 'build', 'wait', 'roll_buildtools', and 'roll_deps'.
  20
  21 The script has a fifth command, 'roll', that runs the four in order.
  22
  23 If given no arguments, the script will run the 'roll' command.
  24
  25 It can only be run on linux in a clean Chromium checkout; it should
  26 error out in most cases if something bad happens, but the error checking
  27 isn't yet foolproof.
  28
  29 """
  30
  31 from __future__ import print_function
  32
  33 import argparse
  34 import json
  35 import os
  36 import re
  37 import subprocess
  38 import sys
  39 import tempfile
  40 import time
  41 import urllib2
  42
  43 depot_tools_path = None
  44 for p in os.environ['PATH'].split(os.pathsep):
  45   if (p.rstrip(os.sep).endswith('depot_tools') and
  46       os.path.isfile(os.path.join(p, 'gclient.py'))):
  47     depot_tools_path = p
  48
  49 assert depot_tools_path
  50 if not depot_tools_path in sys.path:
  51   sys.path.insert(0, depot_tools_path)
  52
  53 third_party_path = os.path.join(depot_tools_path, 'third_party')
  54 if not third_party_path in sys.path:
  55   sys.path.insert(0, third_party_path)
  56
  57 import upload
  58
  59
  60 CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'
  61
  62 CODE_REVIEW_SERVER = 'https://codereview.chromium.org'
  63
  64 COMMITISH_DIGITS = 10
  65
  66 class GNRoller(object):
  67   def __init__(self):
  68     self.chromium_src_dir = None
  69     self.buildtools_dir = None
  70     self.old_gn_commitish = None
  71     self.new_gn_commitish = None
  72     self.old_gn_version   = None
  73     self.new_gn_version = None
  74     self.reviewer = 'dpranke@chromium.org'
  75     if os.getenv('USER') == 'dpranke':
  76       self.reviewer = 'brettw@chromium.org'
  77
  78   def Roll(self):
  79     parser = argparse.ArgumentParser()
  80     parser.usage = __doc__
  81     parser.add_argument('command', nargs='?', default='roll',
  82                         help='build|roll|roll_buildtools|roll_deps|wait'
  83                              ' (%(default)s is the default)')
  84
  85     args = parser.parse_args()
  86     command = args.command
  87     ret = self.SetUp()
  88     if not ret and command in ('roll', 'build'):
  89       ret = self.TriggerBuild()
  90     if not ret and command in ('roll', 'wait'):
  91       ret = self.WaitForBuildToFinish()
  92     if not ret and command in ('roll', 'roll_buildtools'):
  93       ret = self.RollBuildtools()
  94     if not ret and command in ('roll', 'roll_deps'):
  95       ret = self.RollDEPS()
  96
  97     return ret
  98
  99   def SetUp(self):
 100     if sys.platform != 'linux2':
 101       print('roll_gn is only tested and working on Linux for now.')
 102       return 1
 103
 104     ret, out, _ = self.Call('git config --get remote.origin.url')
 105     origin = out.strip()
 106     if ret or origin != CHROMIUM_REPO:
 107       print('Not in a Chromium repo? git config --get remote.origin.url '
 108             'returned %d: %s' % (ret, origin))
 109       return 1
 110
 111     ret, _, _ = self.Call('git diff -q')
 112     if ret:
 113       print("Checkout is dirty, exiting")
 114       return 1
 115
 116     _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
 117     self.chromium_src_dir = out.strip()
 118     self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')
 119
 120     self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()
 121
 122     _, out, _ = self.Call('gn --version')
 123     self.old_gn_version = out.strip()
 124
 125     _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
 126     self.old_gn_commitish = out.strip()
 127     return 0
 128
 129   def GetNewVersions(self):
 130     _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
 131     commit_msg = out.splitlines()
 132     first_line = commit_msg[0]
 133     new_gn_commitish = first_line.split()[1]
 134
 135     last_line = commit_msg[-1]
 136     new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)
 137
 138     return new_gn_commitish, new_gn_version
 139
 140   def TriggerBuild(self):
 141     ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
 142     if ret:
 143       print('Failed to create a new branch for build_gn_%s' %
 144             self.new_gn_version)
 145       return 1
 146
 147     self.MakeDummyDepsChange()
 148
 149     ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
 150                          self.new_gn_version)
 151     if ret:
 152       print('git commit failed: %s' % out + err)
 153       return 1
 154
 155     print('Uploading CL to build GN at {#%s} - %s' %
 156           (self.new_gn_version, self.new_gn_commitish))
 157     ret, out, err = self.Call('git cl upload -f')
 158     if ret:
 159       print('git-cl upload failed: %s' % out + err)
 160       return 1
 161
 162     print('Starting try jobs')
 163     self.Call('git-cl try -m tryserver.chromium.linux '
 164               '-b linux_chromium_gn_upload -r %s' % self.new_gn_commitish)
 165     self.Call('git-cl try -m tryserver.chromium.mac '
 166               '-b mac_chromium_gn_upload -r %s' % self.new_gn_commitish)
 167     self.Call('git-cl try -m tryserver.chromium.win '
 168               '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish)
 169
 170     return 0
 171
 172   def MakeDummyDepsChange(self):
 173     with open('DEPS') as fp:
 174       deps_content = fp.read()
 175       new_deps = deps_content.replace("'buildtools_revision':",
 176                                       "'buildtools_revision':  ")
 177
 178     with open('DEPS', 'w') as fp:
 179       fp.write(new_deps)
 180
 181   def WaitForBuildToFinish(self):
 182     print('Checking build')
 183     results = self.CheckBuild()
 184     while (len(results) or
 185            any(r['state'] == 'pending' for r in results.values())):
 186       print()
 187       print('Sleeping for 30 seconds')
 188       time.sleep(30)
 189       print('Checking build')
 190       results = self.CheckBuild()
 191     return 0 if all(r['state'] == 'success' for r in results.values()) else 1
 192
 193   def CheckBuild(self):
 194     _, out, _ = self.Call('git-cl issue')
 195
 196     issue = int(out.split()[2])
 197
 198     _, out, _ = self.Call('git config user.email')
 199     email = ''
 200     rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email)
 201     try:
 202       props = json.loads(rpc_server.Send('/api/%d' % issue))
 203     except Exception as _e:
 204       raise
 205
 206     patchset = int(props['patchsets'][-1])
 207
 208     try:
 209       patchset_data = json.loads(rpc_server.Send('/api/%d/%d' %
 210                                                  (issue, patchset)))
 211     except Exception as _e:
 212       raise
 213
 214     try_job_results = patchset_data['try_job_results']
 215     if not try_job_results:
 216       print('No try jobs found on most recent patchset')
 217       return {}
 218
 219     results = {}
 220     for job in try_job_results:
 221       builder = job['builder']
 222       if builder == 'linux_chromium_gn_upload':
 223         platform = 'linux64'
 224       elif builder == 'mac_chromium_gn_upload':
 225         platform = 'mac'
 226       elif builder == 'win8_chromium_gn_upload':
 227         platform = 'win'
 228       else:
 229         print('Unexpected builder: %s')
 230         continue
 231
 232       TRY_JOB_RESULT_STATES = ('started', 'success', 'warnings', 'failure',
 233                                'skipped', 'exception', 'retry', 'pending')
 234       state = TRY_JOB_RESULT_STATES[int(job['result']) + 1]
 235       url_str = ' %s' % job['url']
 236       build = url_str.split('/')[-1]
 237
 238       sha1 = '-'
 239       results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str})
 240
 241       if state == 'success':
 242         jsurl = url_str.replace('/builders/', '/json/builders/')
 243         fp = urllib2.urlopen(jsurl)
 244         js = json.loads(fp.read())
 245         fp.close()
 246         for step in js['steps']:
 247           if step['name'] == 'gn sha1':
 248             sha1 = step['text'][1]
 249
 250       if results[platform]['build'] < build:
 251         results[platform]['build'] = build
 252         results[platform]['sha1'] = sha1
 253         results[platform]['state'] = state
 254         results[platform]['url'] = url_str
 255
 256     for platform, r in results.items():
 257       print(platform)
 258       print('  sha1:  %s' % r['sha1'])
 259       print('  state: %s' % r['state'])
 260       print('  build: %s' % r['build'])
 261       print('  url:   %s' % r['url'])
 262       print()
 263
 264     return results
 265
 266   def RollBuildtools(self):
 267     results = self.CheckBuild()
 268     if not all(r['state'] == 'success' for r in results.values()):
 269       print("Roll isn't done or didn't succeed, exiting:")
 270       return 1
 271
 272     desc = self.GetBuildtoolsDesc()
 273
 274     self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
 275               cwd=self.buildtools_dir)
 276
 277     for platform in results:
 278       fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
 279       path = os.path.join(self.buildtools_dir, platform, fname)
 280       with open(path, 'w') as fp:
 281         fp.write('%s\n' % results[platform]['sha1'])
 282
 283     desc_file = tempfile.NamedTemporaryFile(delete=False)
 284     try:
 285       desc_file.write(desc)
 286       desc_file.close()
 287       self.Call('git commit -a -F %s' % desc_file.name,
 288                 cwd=self.buildtools_dir)
 289       self.Call('git-cl upload -f --send-mail',
 290                 cwd=self.buildtools_dir)
 291     finally:
 292       os.remove(desc_file.name)
 293
 294     ret, out, err = self.Call('git cl land', cwd=self.buildtools_dir)
 295     if ret:
 296         print("buildtools git cl land failed: %d" % ret)
 297         if out:
 298             print(out)
 299         if err:
 300             print(err)
 301         return ret
 302
 303     # Fetch the revision we just committed so that RollDEPS will find it.
 304     self.Call('git fetch', cwd=self.buildtools_dir)
 305
 306     # Reset buildtools to the new commit so that we're not still on the
 307     # merged branch.
 308     self.Call('git checkout origin/master', cwd=self.buildtools_dir)
 309
 310     return 0
 311
 312   def RollDEPS(self):
 313     ret, _, _ = self.Call('git new-branch roll_gn_%s' % self.new_gn_version)
 314     if ret:
 315       print('Failed to create a new branch for roll_gn_%s' %
 316             self.new_gn_version)
 317       return 1
 318
 319     _, out, _ = self.Call('git rev-parse origin/master',
 320                           cwd=self.buildtools_dir)
 321     new_buildtools_commitish = out.strip()
 322
 323     new_deps_lines = []
 324     old_buildtools_commitish = ''
 325     with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
 326       for l in fp.readlines():
 327         m = re.match(".*'buildtools_revision':.*'(.+)',", l)
 328         if m:
 329           old_buildtools_commitish = m.group(1)
 330           new_deps_lines.append("  'buildtools_revision': '%s',\n" %
 331                                 new_buildtools_commitish)
 332         else:
 333           new_deps_lines.append(l)
 334
 335     if not old_buildtools_commitish:
 336       print('Could not update DEPS properly, exiting')
 337       return 1
 338
 339     with open('DEPS', 'w') as fp:
 340       fp.write(''.join(new_deps_lines) + '\n')
 341
 342     desc = self.GetDEPSRollDesc(old_buildtools_commitish,
 343                                 new_buildtools_commitish)
 344     desc_file = tempfile.NamedTemporaryFile(delete=False)
 345     try:
 346       desc_file.write(desc)
 347       desc_file.close()
 348       self.Call('git commit -a -F %s' % desc_file.name)
 349       self.Call('git-cl upload -f --send-mail --use-commit-queue')
 350     finally:
 351       os.remove(desc_file.name)
 352
 353     # Intentionally leave the src checkout on the new branch with the roll
 354     # since we're not auto-committing it.
 355
 356     return 0
 357
 358   def GetBuildtoolsDesc(self):
 359     gn_changes = self.GetGNChanges()
 360     return (
 361       'Roll gn %s..%s (r%s:r%s)\n'
 362       '\n'
 363       '%s'
 364       '\n'
 365       'TBR=%s\n' % (
 366         self.old_gn_commitish[:COMMITISH_DIGITS],
 367         self.new_gn_commitish[:COMMITISH_DIGITS],
 368         self.old_gn_version,
 369         self.new_gn_version,
 370         gn_changes,
 371         self.reviewer,
 372       ))
 373
 374   def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
 375     gn_changes = self.GetGNChanges()
 376
 377     return (
 378       'Roll DEPS %s..%s\n'
 379       '\n'
 380       '  In order to roll GN %s..%s (r%s:r%s) and pick up\n'
 381       '  the following changes:\n'
 382       '\n'
 383       '%s'
 384       '\n'
 385       'TBR=%s\n'
 386       'CQ_EXTRA_TRYBOTS=tryserver.chromium.mac:mac_chromium_gn_rel,'
 387       'mac_chromium_gn_dbg;'
 388       'tryserver.chromium.win:win8_chromium_gn_dbg,'
 389       'win_chromium_gn_x64_rel\n' % (
 390         old_buildtools_commitish[:COMMITISH_DIGITS],
 391         new_buildtools_commitish[:COMMITISH_DIGITS],
 392         self.old_gn_commitish[:COMMITISH_DIGITS],
 393         self.new_gn_commitish[:COMMITISH_DIGITS],
 394         self.old_gn_version,
 395         self.new_gn_version,
 396         gn_changes,
 397         self.reviewer,
 398       ))
 399
 400   def GetGNChanges(self):
 401     _, out, _ = self.Call(
 402         "git log --pretty='  %h %s' " +
 403         "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
 404     return out
 405
 406   def Call(self, cmd, cwd=None):
 407     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
 408                             cwd=(cwd or self.chromium_src_dir))
 409     out, err = proc.communicate()
 410     return proc.returncode, out, err
 411
 412
 413 if __name__ == '__main__':
 414   roller = GNRoller()
 415   sys.exit(roller.Roll())