tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import os
  11 import shutil
  12 import stat
  13 import subprocess
  14 import sys
  15
  16 DEFAULT_GCLIENT_CUSTOM_DEPS = {
  17     "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  18                             "chrome/data/page_cycler/.git",
  19     "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  20                          "chrome/data/dom_perf/.git",
  21     "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  22                            "chrome/data/mach_ports/.git",
  23     "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  24                            "chrome/tools/perf/data/.git",
  25     "src/third_party/adobe/flash/binaries/ppapi/linux":
  26         "https://chrome-internal.googlesource.com/"
  27         "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  28     "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  29         "https://chrome-internal.googlesource.com/"
  30         "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  31     "src/third_party/adobe/flash/binaries/ppapi/mac":
  32         "https://chrome-internal.googlesource.com/"
  33         "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  34     "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  35         "https://chrome-internal.googlesource.com/"
  36         "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  37     "src/third_party/adobe/flash/binaries/ppapi/win":
  38         "https://chrome-internal.googlesource.com/"
  39         "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  40     "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  41         "https://chrome-internal.googlesource.com/"
  42         "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",}
  43
  44 GCLIENT_SPEC_DATA = [
  45   { "name"        : "src",
  46     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  47     "deps_file"   : ".DEPS.git",
  48     "managed"     : True,
  49     "custom_deps" : {},
  50     "safesync_url": "",
  51   },
  52 ]
  53 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  54 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  55 FILE_DEPS_GIT = '.DEPS.git'
  56
  57 REPO_PARAMS = [
  58   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  59   '--repo-url',
  60   'https://git.chromium.org/external/repo.git'
  61 ]
  62
  63 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  64                     '--before=%d remotes/m/master)'
  65
  66 ORIGINAL_ENV = {}
  67
  68 def OutputAnnotationStepStart(name):
  69   """Outputs appropriate annotation to signal the start of a step to
  70   a trybot.
  71
  72   Args:
  73     name: The name of the step.
  74   """
  75   print
  76   print '@@@SEED_STEP %s@@@' % name
  77   print '@@@STEP_CURSOR %s@@@' % name
  78   print '@@@STEP_STARTED@@@'
  79   print
  80   sys.stdout.flush()
  81
  82
  83 def OutputAnnotationStepClosed():
  84   """Outputs appropriate annotation to signal the closing of a step to
  85   a trybot."""
  86   print
  87   print '@@@STEP_CLOSED@@@'
  88   print
  89   sys.stdout.flush()
  90
  91
  92 def OutputAnnotationStepLink(label, url):
  93   """Outputs appropriate annotation to print a link.
  94
  95   Args:
  96     label: The name to print.
  97     url: The url to print.
  98   """
  99   print
 100   print '@@@STEP_LINK@%s@%s@@@' % (label, url)
 101   print
 102   sys.stdout.flush()
 103
 104
 105 def IsTelemetryCommand(command):
 106   """Attempts to discern whether or not a given command is running telemetry."""
 107   return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command)
 108
 109
 110 def CreateAndChangeToSourceDirectory(working_directory):
 111   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
 112   the function is successful, the current working directory will change to that
 113   of the new 'bisect' directory.
 114
 115   Returns:
 116     True if the directory was successfully created (or already existed).
 117   """
 118   cwd = os.getcwd()
 119   os.chdir(working_directory)
 120   try:
 121     os.mkdir('bisect')
 122   except OSError, e:
 123     if e.errno != errno.EEXIST:
 124       return False
 125   os.chdir('bisect')
 126   return True
 127
 128
 129 def SubprocessCall(cmd, cwd=None):
 130   """Runs a subprocess with specified parameters.
 131
 132   Args:
 133     params: A list of parameters to pass to gclient.
 134     cwd: Working directory to run from.
 135
 136   Returns:
 137     The return code of the call.
 138   """
 139   if os.name == 'nt':
 140     # "HOME" isn't normally defined on windows, but is needed
 141     # for git to find the user's .netrc file.
 142     if not os.getenv('HOME'):
 143       os.environ['HOME'] = os.environ['USERPROFILE']
 144   shell = os.name == 'nt'
 145   return subprocess.call(cmd, shell=shell, cwd=cwd)
 146
 147
 148 def RunGClient(params, cwd=None):
 149   """Runs gclient with the specified parameters.
 150
 151   Args:
 152     params: A list of parameters to pass to gclient.
 153     cwd: Working directory to run from.
 154
 155   Returns:
 156     The return code of the call.
 157   """
 158   cmd = ['gclient'] + params
 159
 160   return SubprocessCall(cmd, cwd=cwd)
 161
 162
 163 def RunRepo(params):
 164   """Runs cros repo command with specified parameters.
 165
 166   Args:
 167     params: A list of parameters to pass to gclient.
 168
 169   Returns:
 170     The return code of the call.
 171   """
 172   cmd = ['repo'] + params
 173
 174   return SubprocessCall(cmd)
 175
 176
 177 def RunRepoSyncAtTimestamp(timestamp):
 178   """Syncs all git depots to the timestamp specified using repo forall.
 179
 180   Args:
 181     params: Unix timestamp to sync to.
 182
 183   Returns:
 184     The return code of the call.
 185   """
 186   repo_sync = REPO_SYNC_COMMAND % timestamp
 187   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 188   return RunRepo(cmd)
 189
 190
 191 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 192   """Runs gclient and creates a config containing both src and src-internal.
 193
 194   Args:
 195     opts: The options parsed from the command line through parse_args().
 196     custom_deps: A dictionary of additional dependencies to add to .gclient.
 197     cwd: Working directory to run from.
 198
 199   Returns:
 200     The return code of the call.
 201   """
 202   spec = GCLIENT_SPEC_DATA
 203
 204   if custom_deps:
 205     for k, v in custom_deps.iteritems():
 206       spec[0]['custom_deps'][k] = v
 207
 208   # Cannot have newlines in string on windows
 209   spec = 'solutions =' + str(spec)
 210   spec = ''.join([l for l in spec.splitlines()])
 211
 212   if opts.target_platform == 'android':
 213     spec += GCLIENT_SPEC_ANDROID
 214
 215   return_code = RunGClient(
 216       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 217   return return_code
 218
 219
 220 def IsDepsFileBlink():
 221   """Reads .DEPS.git and returns whether or not we're using blink.
 222
 223   Returns:
 224     True if blink, false if webkit.
 225   """
 226   locals = {'Var': lambda _: locals["vars"][_],
 227             'From': lambda *args: None}
 228   execfile(FILE_DEPS_GIT, {}, locals)
 229   return 'blink.git' in locals['vars']['webkit_url']
 230
 231
 232 def RemoveThirdPartyWebkitDirectory():
 233   """Removes third_party/WebKit.
 234
 235   Returns:
 236     True on success.
 237   """
 238   try:
 239     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 240     if os.path.exists(path_to_dir):
 241       shutil.rmtree(path_to_dir)
 242   except OSError, e:
 243     if e.errno != errno.ENOENT:
 244       return False
 245   return True
 246
 247
 248 def OnAccessError(func, path, exc_info):
 249   """
 250   Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
 251
 252   Error handler for ``shutil.rmtree``.
 253
 254   If the error is due to an access error (read only file)
 255   it attempts to add write permission and then retries.
 256
 257   If the error is for another reason it re-raises the error.
 258
 259   Args:
 260     func: The function that raised the error.
 261     path: The path name passed to func.
 262     exc_info: Exception information returned by sys.exc_info().
 263   """
 264   if not os.access(path, os.W_OK):
 265     # Is the error an access error ?
 266     os.chmod(path, stat.S_IWUSR)
 267     func(path)
 268   else:
 269     raise
 270
 271
 272 def RemoveThirdPartyLibjingleDirectory():
 273   """Removes third_party/libjingle. At some point, libjingle was causing issues
 274   syncing when using the git workflow (crbug.com/266324).
 275
 276   Returns:
 277     True on success.
 278   """
 279   path_to_dir = os.path.join(os.getcwd(), 'third_party', 'libjingle')
 280   try:
 281     if os.path.exists(path_to_dir):
 282       shutil.rmtree(path_to_dir, onerror=OnAccessError)
 283   except OSError, e:
 284     print 'Error #%d while running shutil.rmtree(%s): %s' % (
 285         e.errno, path_to_dir, str(e))
 286     if e.errno != errno.ENOENT:
 287       return False
 288   return True
 289
 290
 291 def _CleanupPreviousGitRuns():
 292   """Performs necessary cleanup between runs."""
 293   if os.name != 'nt':
 294     return
 295   # On windows, if a previous run of git crashed, bot was reset, etc... we
 296   # might end up with leftover index.lock files.
 297   for (path, dir, files) in os.walk(os.getcwd()):
 298     for cur_file in files:
 299       if cur_file.endswith('index.lock'):
 300         path_to_file = os.path.join(path, cur_file)
 301         os.remove(path_to_file)
 302
 303
 304 def RunGClientAndSync(cwd=None):
 305   """Runs gclient and does a normal sync.
 306
 307   Args:
 308     cwd: Working directory to run from.
 309
 310   Returns:
 311     The return code of the call.
 312   """
 313   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 314   return RunGClient(params, cwd=cwd)
 315
 316
 317 def SetupGitDepot(opts, custom_deps):
 318   """Sets up the depot for the bisection. The depot will be located in a
 319   subdirectory called 'bisect'.
 320
 321   Args:
 322     opts: The options parsed from the command line through parse_args().
 323     custom_deps: A dictionary of additional dependencies to add to .gclient.
 324
 325   Returns:
 326     True if gclient successfully created the config file and did a sync, False
 327     otherwise.
 328   """
 329   name = 'Setting up Bisection Depot'
 330
 331   if opts.output_buildbot_annotations:
 332     OutputAnnotationStepStart(name)
 333
 334   passed = False
 335
 336   if not RunGClientAndCreateConfig(opts, custom_deps):
 337     passed_deps_check = True
 338     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 339       cwd = os.getcwd()
 340       os.chdir('src')
 341       if not IsDepsFileBlink():
 342         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 343       else:
 344         passed_deps_check = True
 345       if passed_deps_check:
 346         passed_deps_check = RemoveThirdPartyLibjingleDirectory()
 347       os.chdir(cwd)
 348
 349     if passed_deps_check:
 350       _CleanupPreviousGitRuns()
 351
 352       RunGClient(['revert'])
 353       if not RunGClientAndSync():
 354         passed = True
 355
 356   if opts.output_buildbot_annotations:
 357     print
 358     OutputAnnotationStepClosed()
 359
 360   return passed
 361
 362
 363 def SetupCrosRepo():
 364   """Sets up cros repo for bisecting chromeos.
 365
 366   Returns:
 367     Returns 0 on success.
 368   """
 369   cwd = os.getcwd()
 370   try:
 371     os.mkdir('cros')
 372   except OSError, e:
 373     if e.errno != errno.EEXIST:
 374       return False
 375   os.chdir('cros')
 376
 377   cmd = ['init', '-u'] + REPO_PARAMS
 378
 379   passed = False
 380
 381   if not RunRepo(cmd):
 382     if not RunRepo(['sync']):
 383       passed = True
 384   os.chdir(cwd)
 385
 386   return passed
 387
 388
 389 def CopyAndSaveOriginalEnvironmentVars():
 390   """Makes a copy of the current environment variables."""
 391   # TODO: Waiting on crbug.com/255689, will remove this after.
 392   vars_to_remove = []
 393   for k, v in os.environ.iteritems():
 394     if 'ANDROID' in k:
 395       vars_to_remove.append(k)
 396   vars_to_remove.append('CHROME_SRC')
 397   vars_to_remove.append('CHROMIUM_GYP_FILE')
 398   vars_to_remove.append('GYP_CROSSCOMPILE')
 399   vars_to_remove.append('GYP_DEFINES')
 400   vars_to_remove.append('GYP_GENERATORS')
 401   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 402   vars_to_remove.append('OBJCOPY')
 403   for k in vars_to_remove:
 404     if os.environ.has_key(k):
 405       del os.environ[k]
 406
 407   global ORIGINAL_ENV
 408   ORIGINAL_ENV = os.environ.copy()
 409
 410
 411 def SetupAndroidBuildEnvironment(opts, path_to_src=None):
 412   """Sets up the android build environment.
 413
 414   Args:
 415     opts: The options parsed from the command line through parse_args().
 416     path_to_src: Path to the src checkout.
 417
 418   Returns:
 419     True if successful.
 420   """
 421
 422   # Revert the environment variables back to default before setting them up
 423   # with envsetup.sh.
 424   env_vars = os.environ.copy()
 425   for k, _ in env_vars.iteritems():
 426     del os.environ[k]
 427   for k, v in ORIGINAL_ENV.iteritems():
 428     os.environ[k] = v
 429
 430   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 431   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 432                            stdout=subprocess.PIPE,
 433                            stderr=subprocess.PIPE,
 434                            cwd=path_to_src)
 435   (out, _) = proc.communicate()
 436
 437   for line in out.splitlines():
 438     (k, _, v) = line.partition('=')
 439     os.environ[k] = v
 440   return not proc.returncode
 441
 442
 443 def SetupPlatformBuildEnvironment(opts):
 444   """Performs any platform specific setup.
 445
 446   Args:
 447     opts: The options parsed from the command line through parse_args().
 448
 449   Returns:
 450     True if successful.
 451   """
 452   if opts.target_platform == 'android':
 453     CopyAndSaveOriginalEnvironmentVars()
 454     return SetupAndroidBuildEnvironment(opts)
 455   elif opts.target_platform == 'cros':
 456     return SetupCrosRepo()
 457
 458   return True
 459
 460
 461 def CheckIfBisectDepotExists(opts):
 462   """Checks if the bisect directory already exists.
 463
 464   Args:
 465     opts: The options parsed from the command line through parse_args().
 466
 467   Returns:
 468     Returns True if it exists.
 469   """
 470   path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
 471   return os.path.exists(path_to_dir)
 472
 473
 474 def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
 475   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 476   there using gclient.
 477
 478   Args:
 479     opts: The options parsed from the command line through parse_args().
 480     custom_deps: A dictionary of additional dependencies to add to .gclient.
 481   """
 482   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 483     raise RuntimeError('Could not create bisect directory.')
 484
 485   if not SetupGitDepot(opts, custom_deps):
 486     raise RuntimeError('Failed to grab source.')