tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import errno
  17 import json
  18 import os
  19 import pipes
  20 import pprint
  21 import shlex
  22 import shutil
  23 import sys
  24 import subprocess
  25 import tempfile
  26
  27 def main(args):
  28   mbw = MetaBuildWrapper()
  29   mbw.ParseArgs(args)
  30   return mbw.args.func()
  31
  32
  33 class MetaBuildWrapper(object):
  34   def __init__(self):
  35     p = os.path
  36     d = os.path.dirname
  37     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  38     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  39                                  'mb_config.pyl')
  40     self.platform = sys.platform
  41     self.args = argparse.Namespace()
  42     self.configs = {}
  43     self.masters = {}
  44     self.mixins = {}
  45     self.private_configs = []
  46     self.common_dev_configs = []
  47     self.unsupported_configs = []
  48
  49   def ParseArgs(self, argv):
  50     def AddCommonOptions(subp):
  51       subp.add_argument('-b', '--builder',
  52                         help='builder name to look up config from')
  53       subp.add_argument('-m', '--master',
  54                         help='master name to look up config from')
  55       subp.add_argument('-c', '--config',
  56                         help='configuration to analyze')
  57       subp.add_argument('-f', '--config-file', metavar='PATH',
  58                         default=self.default_config,
  59                         help='path to config file '
  60                             '(default is //tools/mb/mb_config.pyl)')
  61       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  62                         help='path to goma directory (default is %(default)s).')
  63       subp.add_argument('-n', '--dryrun', action='store_true',
  64                         help='Do a dry run (i.e., do nothing, just print '
  65                              'the commands that will run)')
  66       subp.add_argument('-q', '--quiet', action='store_true',
  67                         help='Do not print anything on success, '
  68                              'just return an exit code.')
  69       subp.add_argument('-v', '--verbose', action='count',
  70                         help='verbose logging (may specify multiple times).')
  71
  72     parser = argparse.ArgumentParser(prog='mb')
  73     subps = parser.add_subparsers()
  74
  75     subp = subps.add_parser('analyze',
  76                             help='analyze whether changes to a set of files '
  77                                  'will cause a set of binaries to be rebuilt.')
  78     AddCommonOptions(subp)
  79     subp.add_argument('--swarming-targets-file',
  80                       help='save runtime dependencies for targets listed '
  81                            'in file.')
  82     subp.add_argument('path', nargs=1,
  83                       help='path build was generated into.')
  84     subp.add_argument('input_path', nargs=1,
  85                       help='path to a file containing the input arguments '
  86                            'as a JSON object.')
  87     subp.add_argument('output_path', nargs=1,
  88                       help='path to a file containing the output arguments '
  89                            'as a JSON object.')
  90     subp.set_defaults(func=self.CmdAnalyze)
  91
  92     subp = subps.add_parser('gen',
  93                             help='generate a new set of build files')
  94     AddCommonOptions(subp)
  95     subp.add_argument('--swarming-targets-file',
  96                       help='save runtime dependencies for targets listed '
  97                            'in file.')
  98     subp.add_argument('path', nargs=1,
  99                       help='path to generate build into')
 100     subp.set_defaults(func=self.CmdGen)
 101
 102     subp = subps.add_parser('lookup',
 103                             help='look up the command for a given config or '
 104                                  'builder')
 105     AddCommonOptions(subp)
 106     subp.set_defaults(func=self.CmdLookup)
 107
 108     subp = subps.add_parser('validate',
 109                             help='validate the config file')
 110     subp.add_argument('-f', '--config-file', metavar='PATH',
 111                       default=self.default_config,
 112                       help='path to config file '
 113                           '(default is //tools/mb/mb_config.pyl)')
 114     subp.add_argument('-q', '--quiet', action='store_true',
 115                       help='Do not print anything on success, '
 116                            'just return an exit code.')
 117     subp.set_defaults(func=self.CmdValidate)
 118
 119     subp = subps.add_parser('help',
 120                             help='Get help on a subcommand.')
 121     subp.add_argument(nargs='?', action='store', dest='subcommand',
 122                       help='The command to get help for.')
 123     subp.set_defaults(func=self.CmdHelp)
 124
 125     self.args = parser.parse_args(argv)
 126
 127   def CmdAnalyze(self):
 128     vals = self.GetConfig()
 129     if vals['type'] == 'gn':
 130       return self.RunGNAnalyze(vals)
 131     elif vals['type'] == 'gyp':
 132       return self.RunGYPAnalyze(vals)
 133     else:
 134       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 135
 136   def CmdGen(self):
 137     vals = self.GetConfig()
 138
 139     self.ClobberIfNeeded(vals)
 140
 141     if vals['type'] == 'gn':
 142       return self.RunGNGen(vals)
 143     if vals['type'] == 'gyp':
 144       return self.RunGYPGen(vals)
 145
 146     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 147
 148   def CmdLookup(self):
 149     vals = self.GetConfig()
 150     if vals['type'] == 'gn':
 151       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 152     elif vals['type'] == 'gyp':
 153       if vals['gyp_crosscompile']:
 154         self.Print('GYP_CROSSCOMPILE=1')
 155       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
 156     else:
 157       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 158
 159     self.PrintCmd(cmd)
 160     return 0
 161
 162   def CmdHelp(self):
 163     if self.args.subcommand:
 164       self.ParseArgs([self.args.subcommand, '--help'])
 165     else:
 166       self.ParseArgs(['--help'])
 167
 168   def CmdValidate(self):
 169     errs = []
 170
 171     # Read the file to make sure it parses.
 172     self.ReadConfigFile()
 173
 174     # Figure out the whole list of configs and ensure that no config is
 175     # listed in more than one category.
 176     all_configs = {}
 177     for config in self.common_dev_configs:
 178       all_configs[config] = 'common_dev_configs'
 179     for config in self.private_configs:
 180       if config in all_configs:
 181         errs.append('config "%s" listed in "private_configs" also '
 182                     'listed in "%s"' % (config, all_configs['config']))
 183       else:
 184         all_configs[config] = 'private_configs'
 185     for config in self.unsupported_configs:
 186       if config in all_configs:
 187         errs.append('config "%s" listed in "unsupported_configs" also '
 188                     'listed in "%s"' % (config, all_configs['config']))
 189       else:
 190         all_configs[config] = 'unsupported_configs'
 191
 192     for master in self.masters:
 193       for builder in self.masters[master]:
 194         config = self.masters[master][builder]
 195         if config in all_configs and all_configs[config] not in self.masters:
 196           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 197                       (config, all_configs[config]))
 198         else:
 199           all_configs[config] = master
 200
 201     # Check that every referenced config actually exists.
 202     for config, loc in all_configs.items():
 203       if not config in self.configs:
 204         errs.append('Unknown config "%s" referenced from "%s".' %
 205                     (config, loc))
 206
 207     # Check that every actual config is actually referenced.
 208     for config in self.configs:
 209       if not config in all_configs:
 210         errs.append('Unused config "%s".' % config)
 211
 212     # Figure out the whole list of mixins, and check that every mixin
 213     # listed by a config or another mixin actually exists.
 214     referenced_mixins = set()
 215     for config, mixins in self.configs.items():
 216       for mixin in mixins:
 217         if not mixin in self.mixins:
 218           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 219                       (mixin, config))
 220         referenced_mixins.add(mixin)
 221
 222     for mixin in self.mixins:
 223       for sub_mixin in self.mixins[mixin].get('mixins', []):
 224         if not sub_mixin in self.mixins:
 225           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 226                       (sub_mixin, mixin))
 227         referenced_mixins.add(sub_mixin)
 228
 229     # Check that every mixin defined is actually referenced somewhere.
 230     for mixin in self.mixins:
 231       if not mixin in referenced_mixins:
 232         errs.append('Unreferenced mixin "%s".' % mixin)
 233
 234     if errs:
 235       raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
 236                     '\n  ' + '\n  '.join(errs))
 237
 238     if not self.args.quiet:
 239       self.Print('mb config file %s looks ok.' % self.args.config_file)
 240     return 0
 241
 242   def GetConfig(self):
 243     self.ReadConfigFile()
 244     config = self.ConfigFromArgs()
 245     if not config in self.configs:
 246       raise MBErr('Config "%s" not found in %s' %
 247                   (config, self.args.config_file))
 248
 249     return self.FlattenConfig(config)
 250
 251   def ReadConfigFile(self):
 252     if not self.Exists(self.args.config_file):
 253       raise MBErr('config file not found at %s' % self.args.config_file)
 254
 255     try:
 256       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 257     except SyntaxError as e:
 258       raise MBErr('Failed to parse config file "%s": %s' %
 259                  (self.args.config_file, e))
 260
 261     self.common_dev_configs = contents['common_dev_configs']
 262     self.configs = contents['configs']
 263     self.masters = contents['masters']
 264     self.mixins = contents['mixins']
 265     self.private_configs = contents['private_configs']
 266     self.unsupported_configs = contents['unsupported_configs']
 267
 268   def ConfigFromArgs(self):
 269     if self.args.config:
 270       if self.args.master or self.args.builder:
 271         raise MBErr('Can not specific both -c/--config and -m/--master or '
 272                     '-b/--builder')
 273
 274       return self.args.config
 275
 276     if not self.args.master or not self.args.builder:
 277       raise MBErr('Must specify either -c/--config or '
 278                   '(-m/--master and -b/--builder)')
 279
 280     if not self.args.master in self.masters:
 281       raise MBErr('Master name "%s" not found in "%s"' %
 282                   (self.args.master, self.args.config_file))
 283
 284     if not self.args.builder in self.masters[self.args.master]:
 285       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 286                   (self.args.builder, self.args.master, self.args.config_file))
 287
 288     return self.masters[self.args.master][self.args.builder]
 289
 290   def FlattenConfig(self, config):
 291     mixins = self.configs[config]
 292     vals = {
 293       'type': None,
 294       'gn_args': [],
 295       'gyp_config': [],
 296       'gyp_defines': '',
 297       'gyp_crosscompile': False,
 298     }
 299
 300     visited = []
 301     self.FlattenMixins(mixins, vals, visited)
 302     return vals
 303
 304   def FlattenMixins(self, mixins, vals, visited):
 305     for m in mixins:
 306       if m not in self.mixins:
 307         raise MBErr('Unknown mixin "%s"' % m)
 308
 309       # TODO: check for cycles in mixins.
 310
 311       visited.append(m)
 312
 313       mixin_vals = self.mixins[m]
 314       if 'type' in mixin_vals:
 315         vals['type'] = mixin_vals['type']
 316       if 'gn_args' in mixin_vals:
 317         if vals['gn_args']:
 318           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 319         else:
 320           vals['gn_args'] = mixin_vals['gn_args']
 321       if 'gyp_config' in mixin_vals:
 322         vals['gyp_config'] = mixin_vals['gyp_config']
 323       if 'gyp_crosscompile' in mixin_vals:
 324         vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
 325       if 'gyp_defines' in mixin_vals:
 326         if vals['gyp_defines']:
 327           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 328         else:
 329           vals['gyp_defines'] = mixin_vals['gyp_defines']
 330       if 'mixins' in mixin_vals:
 331         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 332     return vals
 333
 334   def ClobberIfNeeded(self, vals):
 335     path = self.args.path[0]
 336     build_dir = self.ToAbsPath(path)
 337     mb_type_path = os.path.join(build_dir, 'mb_type')
 338     needs_clobber = False
 339     new_mb_type = vals['type']
 340     if self.Exists(build_dir):
 341       if self.Exists(mb_type_path):
 342         old_mb_type = self.ReadFile(mb_type_path)
 343         if old_mb_type != new_mb_type:
 344           self.Print("Build type mismatch: was %s, will be %s, clobbering %s" %
 345                      (old_mb_type, new_mb_type, path))
 346           needs_clobber = True
 347       else:
 348         # There is no 'mb_type' file in the build directory, so this probably
 349         # means that the prior build(s) were not done through mb, and we
 350         # have no idea if this was a GYP build or a GN build. Clobber it
 351         # to be safe.
 352         self.Print("%s/mb_type missing, clobbering to be safe" % path)
 353         needs_clobber = True
 354
 355     if needs_clobber:
 356       self.RemoveDirectory(build_dir)
 357
 358     self.MaybeMakeDirectory(build_dir)
 359     self.WriteFile(mb_type_path, new_mb_type)
 360
 361   def RunGNGen(self, vals):
 362     path = self.args.path[0]
 363
 364     cmd = self.GNCmd('gen', path, vals['gn_args'])
 365
 366     swarming_targets = []
 367     if self.args.swarming_targets_file:
 368       # We need GN to generate the list of runtime dependencies for
 369       # the compile targets listed (one per line) in the file so
 370       # we can run them via swarming. We use ninja_to_gn.pyl to convert
 371       # the compile targets to the matching GN labels.
 372       contents = self.ReadFile(self.args.swarming_targets_file)
 373       swarming_targets = contents.splitlines()
 374       gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
 375           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
 376       gn_labels = []
 377       for target in swarming_targets:
 378         if not target in gn_isolate_map:
 379           raise MBErr('test target "%s"  not found in %s' %
 380                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
 381         gn_labels.append(gn_isolate_map[target]['label'])
 382
 383       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 384
 385       # Since GN hasn't run yet, the build directory may not even exist.
 386       self.MaybeMakeDirectory(self.ToAbsPath(path))
 387
 388       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
 389       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 390
 391     ret, _, _ = self.Run(cmd)
 392
 393     for target in swarming_targets:
 394       if gn_isolate_map[target]['type'] == 'gpu_browser_test':
 395         runtime_deps_target = 'browser_tests'
 396       elif gn_isolate_map[target]['type'] == 'script':
 397         # For script targets, the build target is usually a group,
 398         # for which gn generates the runtime_deps next to the stamp file
 399         # for the label, which lives under the obj/ directory.
 400         label = gn_isolate_map[target]['label']
 401         runtime_deps_target = 'obj/%s.stamp' % label.replace(':', '/')
 402       else:
 403         runtime_deps_target = target
 404       if sys.platform == 'win32':
 405         deps_path = self.ToAbsPath(path,
 406                                    runtime_deps_target + '.exe.runtime_deps')
 407       else:
 408         deps_path = self.ToAbsPath(path,
 409                                    runtime_deps_target + '.runtime_deps')
 410       if not self.Exists(deps_path):
 411           raise MBErr('did not generate %s' % deps_path)
 412
 413       command, extra_files = self.GetIsolateCommand(target, vals,
 414                                                     gn_isolate_map)
 415
 416       runtime_deps = self.ReadFile(deps_path).splitlines()
 417
 418       isolate_path = self.ToAbsPath(path, target + '.isolate')
 419       self.WriteFile(isolate_path,
 420         pprint.pformat({
 421           'variables': {
 422             'command': command,
 423             'files': sorted(runtime_deps + extra_files),
 424           }
 425         }) + '\n')
 426
 427       self.WriteJSON(
 428         {
 429           'args': [
 430             '--isolated',
 431             self.ToSrcRelPath('%s%s%s.isolated' % (path, os.sep, target)),
 432             '--isolate',
 433             self.ToSrcRelPath('%s%s%s.isolate' % (path, os.sep, target)),
 434           ],
 435           'dir': self.chromium_src_dir,
 436           'version': 1,
 437         },
 438         isolate_path + 'd.gen.json',
 439       )
 440
 441
 442     return ret
 443
 444   def GNCmd(self, subcommand, path, gn_args=''):
 445     if self.platform == 'linux2':
 446       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
 447                              'gn')
 448     elif self.platform == 'darwin':
 449       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
 450                              'gn')
 451     else:
 452       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
 453                              'gn.exe')
 454
 455     cmd = [gn_path, subcommand, path]
 456     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 457     if gn_args:
 458       cmd.append('--args=%s' % gn_args)
 459     return cmd
 460
 461   def RunGYPGen(self, vals):
 462     path = self.args.path[0]
 463
 464     output_dir, gyp_config = self.ParseGYPConfigPath(path)
 465     if gyp_config != vals['gyp_config']:
 466       raise MBErr('The last component of the path (%s) must match the '
 467                   'GYP configuration specified in the config (%s), and '
 468                   'it does not.' % (gyp_config, vals['gyp_config']))
 469     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 470     env = None
 471     if vals['gyp_crosscompile']:
 472       if self.args.verbose:
 473         self.Print('Setting GYP_CROSSCOMPILE=1 in the environment')
 474       env = os.environ.copy()
 475       env['GYP_CROSSCOMPILE'] = '1'
 476     ret, _, _ = self.Run(cmd, env=env)
 477     return ret
 478
 479   def RunGYPAnalyze(self, vals):
 480     output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
 481     if gyp_config != vals['gyp_config']:
 482       raise MBErr('The last component of the path (%s) must match the '
 483                   'GYP configuration specified in the config (%s), and '
 484                   'it does not.' % (gyp_config, vals['gyp_config']))
 485     if self.args.verbose:
 486       inp = self.ReadInputJSON(['files', 'targets'])
 487       self.Print()
 488       self.Print('analyze input:')
 489       self.PrintJSON(inp)
 490       self.Print()
 491
 492     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 493     cmd.extend(['-f', 'analyzer',
 494                 '-G', 'config_path=%s' % self.args.input_path[0],
 495                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 496     ret, _, _ = self.Run(cmd)
 497     if not ret and self.args.verbose:
 498       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 499       self.Print()
 500       self.Print('analyze output:')
 501       self.PrintJSON(outp)
 502       self.Print()
 503
 504     return ret
 505
 506   def RunGNIsolate(self, vals):
 507     build_path = self.args.path[0]
 508     inp = self.ReadInputJSON(['targets'])
 509     if self.args.verbose:
 510       self.Print()
 511       self.Print('isolate input:')
 512       self.PrintJSON(inp)
 513       self.Print()
 514     output_path = self.args.output_path[0]
 515
 516     for target in inp['targets']:
 517       runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
 518
 519       if not self.Exists(runtime_deps_path):
 520         self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
 521                                   output_path)
 522
 523       command, extra_files = self.GetIsolateCommand(target, vals, None)
 524
 525       runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
 526
 527
 528       isolate_path = self.ToAbsPath(build_path, target + '.isolate')
 529       self.WriteFile(isolate_path,
 530         pprint.pformat({
 531           'variables': {
 532             'command': command,
 533             'files': sorted(runtime_deps + extra_files),
 534           }
 535         }) + '\n')
 536
 537       self.WriteJSON(
 538         {
 539           'args': [
 540             '--isolated',
 541             self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
 542             '--isolate',
 543             self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
 544           ],
 545           'dir': self.chromium_src_dir,
 546           'version': 1,
 547         },
 548         isolate_path + 'd.gen.json',
 549       )
 550
 551     return 0
 552
 553   def GetIsolateCommand(self, target, vals, gn_isolate_map):
 554     # This needs to mirror the settings in //build/config/ui.gni:
 555     # use_x11 = is_linux && !use_ozone.
 556     # TODO(dpranke): Figure out how to keep this in sync better.
 557     use_x11 = (sys.platform == 'linux2' and
 558                not 'target_os="android"' in vals['gn_args'] and
 559                not 'use_ozone=true' in vals['gn_args'])
 560
 561     asan = 'is_asan=true' in vals['gn_args']
 562     msan = 'is_msan=true' in vals['gn_args']
 563     tsan = 'is_tsan=true' in vals['gn_args']
 564
 565     executable_suffix = '.exe' if sys.platform == 'win32' else ''
 566
 567     test_type = gn_isolate_map[target]['type']
 568     cmdline = []
 569     extra_files = []
 570
 571     if use_x11 and test_type == 'windowed_test_launcher':
 572       extra_files = [
 573           'xdisplaycheck',
 574           '../../testing/test_env.py',
 575           '../../testing/xvfb.py',
 576       ]
 577       cmdline = [
 578         '../../testing/xvfb.py',
 579         '.',
 580         './' + str(target),
 581         '--brave-new-test-launcher',
 582         '--test-launcher-bot-mode',
 583         '--asan=%d' % asan,
 584         '--msan=%d' % msan,
 585         '--tsan=%d' % tsan,
 586       ]
 587     elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
 588       extra_files = [
 589           '../../testing/test_env.py'
 590       ]
 591       cmdline = [
 592           '../../testing/test_env.py',
 593           './' + str(target) + executable_suffix,
 594           '--brave-new-test-launcher',
 595           '--test-launcher-bot-mode',
 596           '--asan=%d' % asan,
 597           '--msan=%d' % msan,
 598           '--tsan=%d' % tsan,
 599       ]
 600     elif test_type == 'gpu_browser_test':
 601       extra_files = [
 602           '../../testing/test_env.py'
 603       ]
 604       gtest_filter = gn_isolate_map[target]['gtest_filter']
 605       cmdline = [
 606           '../../testing/test_env.py',
 607           './browser_tests' + executable_suffix,
 608           '--test-launcher-bot-mode',
 609           '--enable-gpu',
 610           '--test-launcher-jobs=1',
 611           '--gtest_filter=%s' % gtest_filter,
 612       ]
 613     elif test_type == 'script':
 614       extra_files = [
 615           '../../testing/test_env.py'
 616       ]
 617       cmdline = [
 618           '../../testing/test_env.py',
 619       ] + ['../../' + self.ToSrcRelPath(gn_isolate_map[target]['script'])]
 620     elif test_type in ('raw'):
 621       extra_files = []
 622       cmdline = [
 623           './' + str(target) + executable_suffix,
 624       ] + gn_isolate_map[target].get('args')
 625
 626     else:
 627       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
 628                                 % (target, test_type), output_path=None)
 629
 630     return cmdline, extra_files
 631
 632   def ToAbsPath(self, build_path, *comps):
 633     return os.path.join(self.chromium_src_dir,
 634                         self.ToSrcRelPath(build_path),
 635                         *comps)
 636
 637   def ToSrcRelPath(self, path):
 638     """Returns a relative path from the top of the repo."""
 639     # TODO: Support normal paths in addition to source-absolute paths.
 640     assert(path.startswith('//'))
 641     return path[2:].replace('/', os.sep)
 642
 643   def ParseGYPConfigPath(self, path):
 644     rpath = self.ToSrcRelPath(path)
 645     output_dir, _, config = rpath.rpartition('/')
 646     self.CheckGYPConfigIsSupported(config, path)
 647     return output_dir, config
 648
 649   def CheckGYPConfigIsSupported(self, config, path):
 650     if config not in ('Debug', 'Release'):
 651       if (sys.platform in ('win32', 'cygwin') and
 652           config not in ('Debug_x64', 'Release_x64')):
 653         raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
 654                     config, path)
 655
 656   def GYPCmd(self, output_dir, gyp_defines, config):
 657     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 658     cmd = [
 659         sys.executable,
 660         os.path.join('build', 'gyp_chromium'),
 661         '-G',
 662         'output_dir=' + output_dir,
 663         '-G',
 664         'config=' + config,
 665     ]
 666     for d in shlex.split(gyp_defines):
 667       cmd += ['-D', d]
 668     return cmd
 669
 670   def RunGNAnalyze(self, vals):
 671     # analyze runs before 'gn gen' now, so we need to run gn gen
 672     # in order to ensure that we have a build directory.
 673     ret = self.RunGNGen(vals)
 674     if ret:
 675       return ret
 676
 677     inp = self.ReadInputJSON(['files', 'targets'])
 678     if self.args.verbose:
 679       self.Print()
 680       self.Print('analyze input:')
 681       self.PrintJSON(inp)
 682       self.Print()
 683
 684     output_path = self.args.output_path[0]
 685
 686     # Bail out early if a GN file was modified, since 'gn refs' won't know
 687     # what to do about it.
 688     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 689       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 690       return 0
 691
 692     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 693     if 'all' in inp['targets']:
 694       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 695       return 0
 696
 697     # This shouldn't normally happen, but could due to unusual race conditions,
 698     # like a try job that gets scheduled before a patch lands but runs after
 699     # the patch has landed.
 700     if not inp['files']:
 701       self.Print('Warning: No files modified in patch, bailing out early.')
 702       self.WriteJSON({'targets': [],
 703                       'build_targets': [],
 704                       'status': 'No dependency'}, output_path)
 705       return 0
 706
 707     ret = 0
 708     response_file = self.TempFile()
 709     response_file.write('\n'.join(inp['files']) + '\n')
 710     response_file.close()
 711
 712     matching_targets = []
 713     try:
 714       cmd = self.GNCmd('refs', self.args.path[0]) + [
 715           '@%s' % response_file.name, '--all', '--as=output']
 716       ret, out, _ = self.Run(cmd)
 717       if ret and not 'The input matches no targets' in out:
 718         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 719                                   output_path)
 720       build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
 721       for output in out.splitlines():
 722         build_output = output.replace(build_dir, '')
 723         if build_output in inp['targets']:
 724           matching_targets.append(build_output)
 725
 726       cmd = self.GNCmd('refs', self.args.path[0]) + [
 727           '@%s' % response_file.name, '--all']
 728       ret, out, _ = self.Run(cmd)
 729       if ret and not 'The input matches no targets' in out:
 730         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 731                                   output_path)
 732       for label in out.splitlines():
 733         build_target = label[2:]
 734         # We want to accept 'chrome/android:chrome_public_apk' and
 735         # just 'chrome_public_apk'. This may result in too many targets
 736         # getting built, but we can adjust that later if need be.
 737         for input_target in inp['targets']:
 738           if (input_target == build_target or
 739               build_target.endswith(':' + input_target)):
 740             matching_targets.append(input_target)
 741     finally:
 742       self.RemoveFile(response_file.name)
 743
 744     if matching_targets:
 745       # TODO: it could be that a target X might depend on a target Y
 746       # and both would be listed in the input, but we would only need
 747       # to specify target X as a build_target (whereas both X and Y are
 748       # targets). I'm not sure if that optimization is generally worth it.
 749       self.WriteJSON({'targets': sorted(matching_targets),
 750                       'build_targets': sorted(matching_targets),
 751                       'status': 'Found dependency'}, output_path)
 752     else:
 753       self.WriteJSON({'targets': [],
 754                       'build_targets': [],
 755                       'status': 'No dependency'}, output_path)
 756
 757     if not ret and self.args.verbose:
 758       outp = json.loads(self.ReadFile(output_path))
 759       self.Print()
 760       self.Print('analyze output:')
 761       self.PrintJSON(outp)
 762       self.Print()
 763
 764     return 0
 765
 766   def ReadInputJSON(self, required_keys):
 767     path = self.args.input_path[0]
 768     output_path = self.args.output_path[0]
 769     if not self.Exists(path):
 770       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 771
 772     try:
 773       inp = json.loads(self.ReadFile(path))
 774     except Exception as e:
 775       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 776                                 (path, e), output_path)
 777
 778     for k in required_keys:
 779       if not k in inp:
 780         self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
 781                                   output_path)
 782
 783     return inp
 784
 785   def WriteFailureAndRaise(self, msg, output_path):
 786     if output_path:
 787       self.WriteJSON({'error': msg}, output_path)
 788     raise MBErr(msg)
 789
 790   def WriteJSON(self, obj, path):
 791     try:
 792       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
 793     except Exception as e:
 794       raise MBErr('Error %s writing to the output path "%s"' %
 795                  (e, path))
 796
 797   def PrintCmd(self, cmd):
 798     if cmd[0] == sys.executable:
 799       cmd = ['python'] + cmd[1:]
 800     self.Print(*[pipes.quote(c) for c in cmd])
 801
 802   def PrintJSON(self, obj):
 803     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 804
 805   def Print(self, *args, **kwargs):
 806     # This function largely exists so it can be overridden for testing.
 807     print(*args, **kwargs)
 808
 809   def Run(self, cmd, env=None):
 810     # This function largely exists so it can be overridden for testing.
 811     if self.args.dryrun or self.args.verbose:
 812       self.PrintCmd(cmd)
 813     if self.args.dryrun:
 814       return 0, '', ''
 815     ret, out, err = self.Call(cmd, env=env)
 816     if self.args.verbose:
 817       if out:
 818         self.Print(out, end='')
 819       if err:
 820         self.Print(err, end='', file=sys.stderr)
 821     return ret, out, err
 822
 823   def Call(self, cmd, env=None):
 824     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 825                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 826                          env=env)
 827     out, err = p.communicate()
 828     return p.returncode, out, err
 829
 830   def ExpandUser(self, path):
 831     # This function largely exists so it can be overridden for testing.
 832     return os.path.expanduser(path)
 833
 834   def Exists(self, path):
 835     # This function largely exists so it can be overridden for testing.
 836     return os.path.exists(path)
 837
 838   def MaybeMakeDirectory(self, path):
 839     try:
 840       os.makedirs(path)
 841     except OSError, e:
 842       if e.errno != errno.EEXIST:
 843         raise
 844
 845   def ReadFile(self, path):
 846     # This function largely exists so it can be overriden for testing.
 847     with open(path) as fp:
 848       return fp.read()
 849
 850   def RemoveFile(self, path):
 851     # This function largely exists so it can be overriden for testing.
 852     os.remove(path)
 853
 854   def RemoveDirectory(self, abs_path):
 855     if sys.platform == 'win32':
 856       # In other places in chromium, we often have to retry this command
 857       # because we're worried about other processes still holding on to
 858       # file handles, but when MB is invoked, it will be early enough in the
 859       # build that their should be no other processes to interfere. We
 860       # can change this if need be.
 861       self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
 862     else:
 863       shutil.rmtree(abs_path, ignore_errors=True)
 864
 865   def TempFile(self, mode='w'):
 866     # This function largely exists so it can be overriden for testing.
 867     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 868
 869   def WriteFile(self, path, contents):
 870     # This function largely exists so it can be overriden for testing.
 871     if self.args.dryrun or self.args.verbose:
 872       self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
 873     with open(path, 'w') as fp:
 874       return fp.write(contents)
 875
 876
 877 class MBErr(Exception):
 878   pass
 879
 880
 881 if __name__ == '__main__':
 882   try:
 883     sys.exit(main(sys.argv[1:]))
 884   except MBErr as e:
 885     print(e)
 886     sys.exit(1)
 887   except KeyboardInterrupt:
 888     print("interrupted, exiting", stream=sys.stderr)
 889     sys.exit(130)