tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import imp
  11 import os
  12 import shutil
  13 import stat
  14 import subprocess
  15 import sys
  16
  17 DEFAULT_GCLIENT_CUSTOM_DEPS = {
  18     "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  19                             "chrome/data/page_cycler/.git",
  20     "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  21                          "chrome/data/dom_perf/.git",
  22     "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  23                            "chrome/data/mach_ports/.git",
  24     "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  25                            "chrome/tools/perf/data/.git",
  26     "src/third_party/adobe/flash/binaries/ppapi/linux":
  27         "https://chrome-internal.googlesource.com/"
  28         "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  29     "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  30         "https://chrome-internal.googlesource.com/"
  31         "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  32     "src/third_party/adobe/flash/binaries/ppapi/mac":
  33         "https://chrome-internal.googlesource.com/"
  34         "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  35     "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  36         "https://chrome-internal.googlesource.com/"
  37         "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  38     "src/third_party/adobe/flash/binaries/ppapi/win":
  39         "https://chrome-internal.googlesource.com/"
  40         "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  41     "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  42         "https://chrome-internal.googlesource.com/"
  43         "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",}
  44
  45 GCLIENT_SPEC_DATA = [
  46   { "name"        : "src",
  47     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  48     "deps_file"   : ".DEPS.git",
  49     "managed"     : True,
  50     "custom_deps" : {},
  51     "safesync_url": "",
  52   },
  53 ]
  54 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  55 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  56 FILE_DEPS_GIT = '.DEPS.git'
  57 FILE_DEPS = 'DEPS'
  58
  59 REPO_PARAMS = [
  60   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  61   '--repo-url',
  62   'https://git.chromium.org/external/repo.git'
  63 ]
  64
  65 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  66                     '--before=%d remotes/m/master)'
  67
  68 ORIGINAL_ENV = {}
  69
  70 def OutputAnnotationStepStart(name):
  71   """Outputs appropriate annotation to signal the start of a step to
  72   a trybot.
  73
  74   Args:
  75     name: The name of the step.
  76   """
  77   print
  78   print '@@@SEED_STEP %s@@@' % name
  79   print '@@@STEP_CURSOR %s@@@' % name
  80   print '@@@STEP_STARTED@@@'
  81   print
  82   sys.stdout.flush()
  83
  84
  85 def OutputAnnotationStepClosed():
  86   """Outputs appropriate annotation to signal the closing of a step to
  87   a trybot."""
  88   print
  89   print '@@@STEP_CLOSED@@@'
  90   print
  91   sys.stdout.flush()
  92
  93
  94 def OutputAnnotationStepLink(label, url):
  95   """Outputs appropriate annotation to print a link.
  96
  97   Args:
  98     label: The name to print.
  99     url: The url to print.
 100   """
 101   print
 102   print '@@@STEP_LINK@%s@%s@@@' % (label, url)
 103   print
 104   sys.stdout.flush()
 105
 106
 107 def LoadExtraSrc(path_to_file):
 108   """Attempts to load an extra source file. If this is successful, uses the
 109   new module to override some global values, such as gclient spec data.
 110
 111   Returns:
 112     The loaded src module, or None."""
 113   try:
 114     global GCLIENT_SPEC_DATA
 115     global GCLIENT_SPEC_ANDROID
 116     extra_src = imp.load_source('data', path_to_file)
 117     GCLIENT_SPEC_DATA = extra_src.GetGClientSpec()
 118     GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams()
 119     return extra_src
 120   except ImportError, e:
 121     return None
 122
 123
 124 def IsTelemetryCommand(command):
 125   """Attempts to discern whether or not a given command is running telemetry."""
 126   return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command)
 127
 128
 129 def CreateAndChangeToSourceDirectory(working_directory):
 130   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
 131   the function is successful, the current working directory will change to that
 132   of the new 'bisect' directory.
 133
 134   Returns:
 135     True if the directory was successfully created (or already existed).
 136   """
 137   cwd = os.getcwd()
 138   os.chdir(working_directory)
 139   try:
 140     os.mkdir('bisect')
 141   except OSError, e:
 142     if e.errno != errno.EEXIST:
 143       return False
 144   os.chdir('bisect')
 145   return True
 146
 147
 148 def SubprocessCall(cmd, cwd=None):
 149   """Runs a subprocess with specified parameters.
 150
 151   Args:
 152     params: A list of parameters to pass to gclient.
 153     cwd: Working directory to run from.
 154
 155   Returns:
 156     The return code of the call.
 157   """
 158   if os.name == 'nt':
 159     # "HOME" isn't normally defined on windows, but is needed
 160     # for git to find the user's .netrc file.
 161     if not os.getenv('HOME'):
 162       os.environ['HOME'] = os.environ['USERPROFILE']
 163   shell = os.name == 'nt'
 164   return subprocess.call(cmd, shell=shell, cwd=cwd)
 165
 166
 167 def RunGClient(params, cwd=None):
 168   """Runs gclient with the specified parameters.
 169
 170   Args:
 171     params: A list of parameters to pass to gclient.
 172     cwd: Working directory to run from.
 173
 174   Returns:
 175     The return code of the call.
 176   """
 177   cmd = ['gclient'] + params
 178
 179   return SubprocessCall(cmd, cwd=cwd)
 180
 181
 182 def RunRepo(params):
 183   """Runs cros repo command with specified parameters.
 184
 185   Args:
 186     params: A list of parameters to pass to gclient.
 187
 188   Returns:
 189     The return code of the call.
 190   """
 191   cmd = ['repo'] + params
 192
 193   return SubprocessCall(cmd)
 194
 195
 196 def RunRepoSyncAtTimestamp(timestamp):
 197   """Syncs all git depots to the timestamp specified using repo forall.
 198
 199   Args:
 200     params: Unix timestamp to sync to.
 201
 202   Returns:
 203     The return code of the call.
 204   """
 205   repo_sync = REPO_SYNC_COMMAND % timestamp
 206   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 207   return RunRepo(cmd)
 208
 209
 210 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 211   """Runs gclient and creates a config containing both src and src-internal.
 212
 213   Args:
 214     opts: The options parsed from the command line through parse_args().
 215     custom_deps: A dictionary of additional dependencies to add to .gclient.
 216     cwd: Working directory to run from.
 217
 218   Returns:
 219     The return code of the call.
 220   """
 221   spec = GCLIENT_SPEC_DATA
 222
 223   if custom_deps:
 224     for k, v in custom_deps.iteritems():
 225       spec[0]['custom_deps'][k] = v
 226
 227   # Cannot have newlines in string on windows
 228   spec = 'solutions =' + str(spec)
 229   spec = ''.join([l for l in spec.splitlines()])
 230
 231   if 'android' in opts.target_platform:
 232     spec += GCLIENT_SPEC_ANDROID
 233
 234   return_code = RunGClient(
 235       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 236   return return_code
 237
 238
 239 def IsDepsFileBlink():
 240   """Reads .DEPS.git and returns whether or not we're using blink.
 241
 242   Returns:
 243     True if blink, false if webkit.
 244   """
 245   locals = {'Var': lambda _: locals["vars"][_],
 246             'From': lambda *args: None}
 247   execfile(FILE_DEPS_GIT, {}, locals)
 248   return 'blink.git' in locals['vars']['webkit_url']
 249
 250
 251 def RemoveThirdPartyWebkitDirectory():
 252   """Removes third_party/WebKit.
 253
 254   Returns:
 255     True on success.
 256   """
 257   try:
 258     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 259     if os.path.exists(path_to_dir):
 260       shutil.rmtree(path_to_dir)
 261   except OSError, e:
 262     if e.errno != errno.ENOENT:
 263       return False
 264   return True
 265
 266
 267 def OnAccessError(func, path, exc_info):
 268   """
 269   Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
 270
 271   Error handler for ``shutil.rmtree``.
 272
 273   If the error is due to an access error (read only file)
 274   it attempts to add write permission and then retries.
 275
 276   If the error is for another reason it re-raises the error.
 277
 278   Args:
 279     func: The function that raised the error.
 280     path: The path name passed to func.
 281     exc_info: Exception information returned by sys.exc_info().
 282   """
 283   if not os.access(path, os.W_OK):
 284     # Is the error an access error ?
 285     os.chmod(path, stat.S_IWUSR)
 286     func(path)
 287   else:
 288     raise
 289
 290
 291 def RemoveThirdPartyLibjingleDirectory():
 292   """Removes third_party/libjingle. At some point, libjingle was causing issues
 293   syncing when using the git workflow (crbug.com/266324).
 294
 295   Returns:
 296     True on success.
 297   """
 298   path_to_dir = os.path.join(os.getcwd(), 'third_party', 'libjingle')
 299   try:
 300     if os.path.exists(path_to_dir):
 301       shutil.rmtree(path_to_dir, onerror=OnAccessError)
 302   except OSError, e:
 303     print 'Error #%d while running shutil.rmtree(%s): %s' % (
 304         e.errno, path_to_dir, str(e))
 305     if e.errno != errno.ENOENT:
 306       return False
 307   return True
 308
 309
 310 def _CleanupPreviousGitRuns():
 311   """Performs necessary cleanup between runs."""
 312   # If a previous run of git crashed, bot was reset, etc... we
 313   # might end up with leftover index.lock files.
 314   for (path, dir, files) in os.walk(os.getcwd()):
 315     for cur_file in files:
 316       if cur_file.endswith('index.lock'):
 317         path_to_file = os.path.join(path, cur_file)
 318         os.remove(path_to_file)
 319
 320
 321 def RunGClientAndSync(cwd=None):
 322   """Runs gclient and does a normal sync.
 323
 324   Args:
 325     cwd: Working directory to run from.
 326
 327   Returns:
 328     The return code of the call.
 329   """
 330   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 331   return RunGClient(params, cwd=cwd)
 332
 333
 334 def SetupGitDepot(opts, custom_deps):
 335   """Sets up the depot for the bisection. The depot will be located in a
 336   subdirectory called 'bisect'.
 337
 338   Args:
 339     opts: The options parsed from the command line through parse_args().
 340     custom_deps: A dictionary of additional dependencies to add to .gclient.
 341
 342   Returns:
 343     True if gclient successfully created the config file and did a sync, False
 344     otherwise.
 345   """
 346   name = 'Setting up Bisection Depot'
 347
 348   if opts.output_buildbot_annotations:
 349     OutputAnnotationStepStart(name)
 350
 351   passed = False
 352
 353   if not RunGClientAndCreateConfig(opts, custom_deps):
 354     passed_deps_check = True
 355     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 356       cwd = os.getcwd()
 357       os.chdir('src')
 358       if not IsDepsFileBlink():
 359         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 360       else:
 361         passed_deps_check = True
 362       if passed_deps_check:
 363         passed_deps_check = RemoveThirdPartyLibjingleDirectory()
 364       os.chdir(cwd)
 365
 366     if passed_deps_check:
 367       _CleanupPreviousGitRuns()
 368
 369       RunGClient(['revert'])
 370       if not RunGClientAndSync():
 371         passed = True
 372
 373   if opts.output_buildbot_annotations:
 374     print
 375     OutputAnnotationStepClosed()
 376
 377   return passed
 378
 379
 380 def SetupCrosRepo():
 381   """Sets up cros repo for bisecting chromeos.
 382
 383   Returns:
 384     Returns 0 on success.
 385   """
 386   cwd = os.getcwd()
 387   try:
 388     os.mkdir('cros')
 389   except OSError, e:
 390     if e.errno != errno.EEXIST:
 391       return False
 392   os.chdir('cros')
 393
 394   cmd = ['init', '-u'] + REPO_PARAMS
 395
 396   passed = False
 397
 398   if not RunRepo(cmd):
 399     if not RunRepo(['sync']):
 400       passed = True
 401   os.chdir(cwd)
 402
 403   return passed
 404
 405
 406 def CopyAndSaveOriginalEnvironmentVars():
 407   """Makes a copy of the current environment variables."""
 408   # TODO: Waiting on crbug.com/255689, will remove this after.
 409   vars_to_remove = []
 410   for k, v in os.environ.iteritems():
 411     if 'ANDROID' in k:
 412       vars_to_remove.append(k)
 413   vars_to_remove.append('CHROME_SRC')
 414   vars_to_remove.append('CHROMIUM_GYP_FILE')
 415   vars_to_remove.append('GYP_CROSSCOMPILE')
 416   vars_to_remove.append('GYP_DEFINES')
 417   vars_to_remove.append('GYP_GENERATORS')
 418   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 419   vars_to_remove.append('OBJCOPY')
 420   for k in vars_to_remove:
 421     if os.environ.has_key(k):
 422       del os.environ[k]
 423
 424   global ORIGINAL_ENV
 425   ORIGINAL_ENV = os.environ.copy()
 426
 427
 428 def SetupAndroidBuildEnvironment(opts, path_to_src=None):
 429   """Sets up the android build environment.
 430
 431   Args:
 432     opts: The options parsed from the command line through parse_args().
 433     path_to_src: Path to the src checkout.
 434
 435   Returns:
 436     True if successful.
 437   """
 438
 439   # Revert the environment variables back to default before setting them up
 440   # with envsetup.sh.
 441   env_vars = os.environ.copy()
 442   for k, _ in env_vars.iteritems():
 443     del os.environ[k]
 444   for k, v in ORIGINAL_ENV.iteritems():
 445     os.environ[k] = v
 446
 447   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 448   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 449                            stdout=subprocess.PIPE,
 450                            stderr=subprocess.PIPE,
 451                            cwd=path_to_src)
 452   (out, _) = proc.communicate()
 453
 454   for line in out.splitlines():
 455     (k, _, v) = line.partition('=')
 456     os.environ[k] = v
 457
 458   return not proc.returncode
 459
 460
 461 def SetupPlatformBuildEnvironment(opts):
 462   """Performs any platform specific setup.
 463
 464   Args:
 465     opts: The options parsed from the command line through parse_args().
 466
 467   Returns:
 468     True if successful.
 469   """
 470   if 'android' in opts.target_platform:
 471     CopyAndSaveOriginalEnvironmentVars()
 472     return SetupAndroidBuildEnvironment(opts)
 473   elif opts.target_platform == 'cros':
 474     return SetupCrosRepo()
 475
 476   return True
 477
 478
 479 def CheckIfBisectDepotExists(opts):
 480   """Checks if the bisect directory already exists.
 481
 482   Args:
 483     opts: The options parsed from the command line through parse_args().
 484
 485   Returns:
 486     Returns True if it exists.
 487   """
 488   path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
 489   return os.path.exists(path_to_dir)
 490
 491
 492 def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
 493   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 494   there using gclient.
 495
 496   Args:
 497     opts: The options parsed from the command line through parse_args().
 498     custom_deps: A dictionary of additional dependencies to add to .gclient.
 499   """
 500   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 501     raise RuntimeError('Could not create bisect directory.')
 502
 503   if not SetupGitDepot(opts, custom_deps):
 504     raise RuntimeError('Failed to grab source.')