tools/gn/bin/roll_gn.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """An auto-roller for GN binaries into Chromium.
   6
   7 This script is used to update the GN binaries that a Chromium
   8 checkout uses. In order to update the binaries, one must follow
   9 four steps in order:
  10
  11 1. Trigger try jobs to build a new GN binary at tip-of-tree and upload
  12    the newly-built binaries into the right Google CloudStorage bucket.
  13 2. Wait for the try jobs to complete.
  14 3. Update the buildtools repo with the .sha1 hashes of the newly built
  15    binaries.
  16 4. Update Chromium's DEPS file to the new version of the buildtools repo.
  17
  18 The script has four commands that correspond to the four steps above:
  19 'build', 'wait', 'roll_buildtools', and 'roll_deps'.
  20
  21 The script has a fifth command, 'roll', that runs the four in order.
  22
  23 If given no arguments, the script will run the 'roll' command.
  24
  25 It can only be run on linux in a clean Chromium checkout; it should
  26 error out in most cases if something bad happens, but the error checking
  27 isn't yet foolproof.
  28
  29 """
  30
  31 from __future__ import print_function
  32
  33 import argparse
  34 import json
  35 import os
  36 import re
  37 import subprocess
  38 import sys
  39 import tempfile
  40 import time
  41 import urllib2
  42
  43 depot_tools_path = None
  44 for p in os.environ['PATH'].split(os.pathsep):
  45   if (p.rstrip(os.sep).endswith('depot_tools') and
  46       os.path.isfile(os.path.join(p, 'gclient.py'))):
  47     depot_tools_path = p
  48
  49 assert depot_tools_path
  50 if not depot_tools_path in sys.path:
  51   sys.path.insert(0, depot_tools_path)
  52
  53 third_party_path = os.path.join(depot_tools_path, 'third_party')
  54 if not third_party_path in sys.path:
  55   sys.path.insert(0, third_party_path)
  56
  57 import upload
  58
  59
  60 CHROMIUM_REPO = 'https://chromium.googlesource.com/chromium/src.git'
  61
  62 CODE_REVIEW_SERVER = 'https://codereview.chromium.org'
  63
  64
  65 class GNRoller(object):
  66   def __init__(self):
  67     self.chromium_src_dir = None
  68     self.buildtools_dir = None
  69     self.old_gn_commitish = None
  70     self.new_gn_commitish = None
  71     self.old_gn_version   = None
  72     self.new_gn_version = None
  73     self.reviewer = 'dpranke@chromium.org'
  74     if os.getenv('USER') == 'dpranke':
  75       self.reviewer = 'brettw@chromium.org'
  76
  77   def Roll(self):
  78     parser = argparse.ArgumentParser()
  79     parser.usage = __doc__
  80     parser.add_argument('command', nargs='?', default='roll',
  81                         help='build|roll|roll_buildtools|roll_deps|wait'
  82                              ' (%(default)s is the default)')
  83
  84     args = parser.parse_args()
  85     command = args.command
  86     ret = self.SetUp()
  87     if not ret and command in ('roll', 'build'):
  88       ret = self.TriggerBuild()
  89     if not ret and command in ('roll', 'wait'):
  90       ret = self.WaitForBuildToFinish()
  91     if not ret and command in ('roll', 'roll_buildtools'):
  92       ret = self.RollBuildtools()
  93     if not ret and command in ('roll', 'roll_deps'):
  94       ret = self.RollDEPS()
  95
  96     return ret
  97
  98   def SetUp(self):
  99     if sys.platform != 'linux2':
 100       print('roll_gn is only tested and working on Linux for now.')
 101       return 1
 102
 103     ret, out, _ = self.Call('git config --get remote.origin.url')
 104     origin = out.strip()
 105     if ret or origin != CHROMIUM_REPO:
 106       print('Not in a Chromium repo? git config --get remote.origin.url '
 107             'returned %d: %s' % (ret, origin))
 108       return 1
 109
 110     ret, _, _ = self.Call('git diff -q')
 111     if ret:
 112       print("Checkout is dirty, exiting")
 113       return 1
 114
 115     _, out, _ = self.Call('git rev-parse --show-toplevel', cwd=os.getcwd())
 116     self.chromium_src_dir = out.strip()
 117     self.buildtools_dir = os.path.join(self.chromium_src_dir, 'buildtools')
 118
 119     self.new_gn_commitish, self.new_gn_version = self.GetNewVersions()
 120
 121     _, out, _ = self.Call('gn --version')
 122     self.old_gn_version = out.strip()
 123
 124     _, out, _ = self.Call('git crrev-parse %s' % self.old_gn_version)
 125     self.old_gn_commitish = out.strip()
 126     return 0
 127
 128   def GetNewVersions(self):
 129     _, out, _ = self.Call('git log -1 --grep Cr-Commit-Position')
 130     commit_msg = out.splitlines()
 131     first_line = commit_msg[0]
 132     new_gn_commitish = first_line.split()[1]
 133
 134     last_line = commit_msg[-1]
 135     new_gn_version = re.sub('.*master@{#(\d+)}', '\\1', last_line)
 136
 137     return new_gn_commitish, new_gn_version
 138
 139   def TriggerBuild(self):
 140     ret, _, _ = self.Call('git new-branch build_gn_%s' % self.new_gn_version)
 141     if ret:
 142       print('Failed to create a new branch for build_gn_%s' %
 143             self.new_gn_version)
 144       return 1
 145
 146     self.MakeDummyDepsChange()
 147
 148     ret, out, err = self.Call('git commit -a -m "Build gn at %s"' %
 149                          self.new_gn_version)
 150     if ret:
 151       print('git commit failed: %s' % out + err)
 152       return 1
 153
 154     print('Uploading CL to build GN at {#%s} - %s' %
 155           (self.new_gn_version, self.new_gn_commitish))
 156     ret, out, err = self.Call('git cl upload -f')
 157     if ret:
 158       print('git-cl upload failed: %s' % out + err)
 159       return 1
 160
 161     print('Starting try jobs')
 162     self.Call('git-cl try -b linux_chromium_gn_upload '
 163               '-b mac_chromium_gn_upload '
 164               '-b win8_chromium_gn_upload -r %s' % self.new_gn_commitish)
 165
 166     return 0
 167
 168   def MakeDummyDepsChange(self):
 169     with open('DEPS') as fp:
 170       deps_content = fp.read()
 171       new_deps = deps_content.replace("'buildtools_revision':",
 172                                       "'buildtools_revision':  ")
 173
 174     with open('DEPS', 'w') as fp:
 175       fp.write(new_deps)
 176
 177   def WaitForBuildToFinish(self):
 178     print('Checking build')
 179     results = self.CheckBuild()
 180     while any(r['state'] == 'pending' for r in results.values()):
 181       print()
 182       print('Sleeping for 30 seconds')
 183       time.sleep(30)
 184       print('Checking build')
 185       results = self.CheckBuild()
 186     return 0 if all(r['state'] == 'success' for r in results.values()) else 1
 187
 188   def CheckBuild(self):
 189     _, out, _ = self.Call('git-cl issue')
 190
 191     issue = int(out.split()[2])
 192
 193     _, out, _ = self.Call('git config user.email')
 194     email = ''
 195     rpc_server = upload.GetRpcServer(CODE_REVIEW_SERVER, email)
 196     try:
 197       props = json.loads(rpc_server.Send('/api/%d' % issue))
 198     except Exception as _e:
 199       raise
 200
 201     patchset = int(props['patchsets'][-1])
 202
 203     try:
 204       patchset_data = json.loads(rpc_server.Send('/api/%d/%d' %
 205                                                  (issue, patchset)))
 206     except Exception as _e:
 207       raise
 208
 209     TRY_JOB_RESULT_STATES = ('success', 'warnings', 'failure', 'skipped',
 210                              'exception', 'retry', 'pending')
 211     try_job_results = patchset_data['try_job_results']
 212     if not try_job_results:
 213       print('No try jobs found on most recent patchset')
 214       return 1
 215
 216     results = {}
 217     for job in try_job_results:
 218       builder = job['builder']
 219       if builder == 'linux_chromium_gn_upload':
 220         platform = 'linux64'
 221       elif builder == 'mac_chromium_gn_upload':
 222         platform = 'mac'
 223       elif builder == 'win8_chromium_gn_upload':
 224         platform = 'win'
 225       else:
 226         print('Unexpected builder: %s')
 227         continue
 228
 229       state = TRY_JOB_RESULT_STATES[int(job['result'])]
 230       url_str = ' %s' % job['url']
 231       build = url_str.split('/')[-1]
 232
 233       sha1 = '-'
 234       results.setdefault(platform, {'build': -1, 'sha1': '', 'url': url_str})
 235
 236       if state == 'success':
 237         jsurl = url_str.replace('/builders/', '/json/builders/')
 238         fp = urllib2.urlopen(jsurl)
 239         js = json.loads(fp.read())
 240         fp.close()
 241         for step in js['steps']:
 242           if step['name'] == 'gn sha1':
 243             sha1 = step['text'][1]
 244
 245       if results[platform]['build'] < build:
 246         results[platform]['build'] = build
 247         results[platform]['sha1'] = sha1
 248         results[platform]['state'] = state
 249         results[platform]['url'] = url_str
 250
 251     for platform, r in results.items():
 252       print(platform)
 253       print('  sha1:  %s' % r['sha1'])
 254       print('  state: %s' % r['state'])
 255       print('  build: %s' % r['build'])
 256       print('  url:   %s' % r['url'])
 257       print()
 258
 259     return results
 260
 261   def RollBuildtools(self):
 262     results = self.CheckBuild()
 263     if not all(r['state'] == 'success' for r in results.values()):
 264       print("Roll isn't done or didn't succeed, exiting:")
 265       return 1
 266
 267     desc = self.GetBuildtoolsDesc()
 268
 269     self.Call('git new-branch roll_buildtools_gn_%s' % self.new_gn_version,
 270               cwd=self.buildtools_dir)
 271
 272     for platform in results:
 273       fname = 'gn.exe.sha1' if platform == 'win' else 'gn.sha1'
 274       path = os.path.join(self.buildtools_dir, platform, fname)
 275       with open(path, 'w') as fp:
 276         fp.write('%s\n' % results[platform]['sha1'])
 277
 278     desc_file = tempfile.NamedTemporaryFile(delete=False)
 279     try:
 280       desc_file.write(desc)
 281       desc_file.close()
 282       self.Call('git commit -a -F %s' % desc_file.name,
 283                 cwd=self.buildtools_dir)
 284       self.Call('git-cl upload -f --send-mail',
 285                 cwd=self.buildtools_dir)
 286     finally:
 287       os.remove(desc_file.name)
 288
 289     self.Call('git cl push', cwd=self.buildtools_dir)
 290
 291     # Fetch the revision we just committed so that RollDEPS will find it.
 292     self.Call('git cl fetch', cwd=self.buildtools_dir)
 293
 294     return 0
 295
 296   def RollDEPS(self):
 297     _, out, _ = self.Call('git rev-parse origin/master',
 298                           cwd=self.buildtools_dir)
 299     new_buildtools_commitish = out.strip()
 300
 301     new_deps_lines = []
 302     old_buildtools_commitish = ''
 303     with open(os.path.join(self.chromium_src_dir, 'DEPS')) as fp:
 304       for l in fp.readlines():
 305         m = re.match(".*'buildtools_revision':.*'(.+)',", l)
 306         if m:
 307           old_buildtools_commitish = m.group(1)
 308           new_deps_lines.append("  'buildtools_revision': '%s'," %
 309                                 new_buildtools_commitish)
 310         else:
 311           new_deps_lines.append(l)
 312
 313     if not old_buildtools_commitish:
 314       print('Could not update DEPS properly, exiting')
 315       return 1
 316
 317     with open('DEPS', 'w') as fp:
 318       fp.write(''.join(new_deps_lines) + '\n')
 319
 320     desc = self.GetDEPSRollDesc(old_buildtools_commitish,
 321                                 new_buildtools_commitish)
 322     desc_file = tempfile.NamedTemporaryFile(delete=False)
 323     try:
 324       desc_file.write(desc)
 325       desc_file.close()
 326       self.Call('git commit -a -F %s' % desc_file.name)
 327       self.Call('git-cl upload -f --send-mail --commit-queue')
 328     finally:
 329       os.remove(desc_file.name)
 330     return 0
 331
 332   def GetBuildtoolsDesc(self):
 333     gn_changes = self.GetGNChanges()
 334     return (
 335       'Roll gn %s..%s (r%s:%s)\n'
 336       '\n'
 337       '%s'
 338       '\n'
 339       'TBR=%s\n' % (
 340         self.old_gn_commitish,
 341         self.new_gn_commitish,
 342         self.old_gn_version,
 343         self.new_gn_version,
 344         gn_changes,
 345         self.reviewer,
 346       ))
 347
 348   def GetDEPSRollDesc(self, old_buildtools_commitish, new_buildtools_commitish):
 349     gn_changes = self.GetGNChanges()
 350
 351     return (
 352       'Roll DEPS %s..%s\n'
 353       '\n'
 354       '  in order to roll GN %s..%s (r%s:%s)\n'
 355       '\n'
 356       '%s'
 357       '\n'
 358       'TBR=%s\n' % (
 359         old_buildtools_commitish,
 360         new_buildtools_commitish,
 361         self.old_gn_commitish,
 362         self.new_gn_commitish,
 363         self.old_gn_version,
 364         self.new_gn_version,
 365         gn_changes,
 366         self.reviewer,
 367       ))
 368
 369   def GetGNChanges(self):
 370     _, out, _ = self.Call(
 371         "git log --pretty='  %h %s' " +
 372         "%s..%s tools/gn" % (self.old_gn_commitish, self.new_gn_commitish))
 373     return out
 374
 375   def Call(self, cmd, cwd=None):
 376     proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True,
 377                             cwd=(cwd or self.chromium_src_dir))
 378     out, err = proc.communicate()
 379     return proc.returncode, out, err
 380
 381
 382 if __name__ == '__main__':
 383   roller = GNRoller()
 384   sys.exit(roller.Roll())