tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import errno
  17 import json
  18 import os
  19 import pprint
  20 import shutil
  21 import sys
  22 import subprocess
  23 import tempfile
  24
  25 def main(args):
  26   mbw = MetaBuildWrapper()
  27   mbw.ParseArgs(args)
  28   return mbw.args.func()
  29
  30
  31 class MetaBuildWrapper(object):
  32   def __init__(self):
  33     p = os.path
  34     d = os.path.dirname
  35     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  36     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  37                                  'mb_config.pyl')
  38     self.executable = sys.executable
  39     self.platform = sys.platform
  40     self.sep = os.sep
  41     self.args = argparse.Namespace()
  42     self.configs = {}
  43     self.masters = {}
  44     self.mixins = {}
  45     self.private_configs = []
  46     self.common_dev_configs = []
  47     self.unsupported_configs = []
  48
  49   def ParseArgs(self, argv):
  50     def AddCommonOptions(subp):
  51       subp.add_argument('-b', '--builder',
  52                         help='builder name to look up config from')
  53       subp.add_argument('-m', '--master',
  54                         help='master name to look up config from')
  55       subp.add_argument('-c', '--config',
  56                         help='configuration to analyze')
  57       subp.add_argument('-f', '--config-file', metavar='PATH',
  58                         default=self.default_config,
  59                         help='path to config file '
  60                             '(default is //tools/mb/mb_config.pyl)')
  61       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  62                         help='path to goma directory (default is %(default)s).')
  63       subp.add_argument('-n', '--dryrun', action='store_true',
  64                         help='Do a dry run (i.e., do nothing, just print '
  65                              'the commands that will run)')
  66       subp.add_argument('-v', '--verbose', action='store_true',
  67                         help='verbose logging')
  68
  69     parser = argparse.ArgumentParser(prog='mb')
  70     subps = parser.add_subparsers()
  71
  72     subp = subps.add_parser('analyze',
  73                             help='analyze whether changes to a set of files '
  74                                  'will cause a set of binaries to be rebuilt.')
  75     AddCommonOptions(subp)
  76     subp.add_argument('--swarming-targets-file',
  77                       help='save runtime dependencies for targets listed '
  78                            'in file.')
  79     subp.add_argument('path', nargs=1,
  80                       help='path build was generated into.')
  81     subp.add_argument('input_path', nargs=1,
  82                       help='path to a file containing the input arguments '
  83                            'as a JSON object.')
  84     subp.add_argument('output_path', nargs=1,
  85                       help='path to a file containing the output arguments '
  86                            'as a JSON object.')
  87     subp.set_defaults(func=self.CmdAnalyze)
  88
  89     subp = subps.add_parser('gen',
  90                             help='generate a new set of build files')
  91     AddCommonOptions(subp)
  92     subp.add_argument('--swarming-targets-file',
  93                       help='save runtime dependencies for targets listed '
  94                            'in file.')
  95     subp.add_argument('path', nargs=1,
  96                       help='path to generate build into')
  97     subp.set_defaults(func=self.CmdGen)
  98
  99     subp = subps.add_parser('lookup',
 100                             help='look up the command for a given config or '
 101                                  'builder')
 102     AddCommonOptions(subp)
 103     subp.set_defaults(func=self.CmdLookup)
 104
 105     subp = subps.add_parser('validate',
 106                             help='validate the config file')
 107     subp.add_argument('-f', '--config-file', metavar='PATH',
 108                       default=self.default_config,
 109                       help='path to config file '
 110                           '(default is //tools/mb/mb_config.pyl)')
 111     subp.set_defaults(func=self.CmdValidate)
 112
 113     subp = subps.add_parser('help',
 114                             help='Get help on a subcommand.')
 115     subp.add_argument(nargs='?', action='store', dest='subcommand',
 116                       help='The command to get help for.')
 117     subp.set_defaults(func=self.CmdHelp)
 118
 119     self.args = parser.parse_args(argv)
 120
 121   def CmdAnalyze(self):
 122     vals = self.GetConfig()
 123     if vals['type'] == 'gn':
 124       return self.RunGNAnalyze(vals)
 125     elif vals['type'] == 'gyp':
 126       return self.RunGYPAnalyze(vals)
 127     else:
 128       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 129
 130   def CmdGen(self):
 131     vals = self.GetConfig()
 132
 133     self.ClobberIfNeeded(vals)
 134
 135     if vals['type'] == 'gn':
 136       return self.RunGNGen(vals)
 137     if vals['type'] == 'gyp':
 138       return self.RunGYPGen(vals)
 139
 140     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 141
 142   def CmdLookup(self):
 143     vals = self.GetConfig()
 144     if vals['type'] == 'gn':
 145       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 146     elif vals['type'] == 'gyp':
 147       if vals['gyp_crosscompile']:
 148         self.Print('GYP_CROSSCOMPILE=1')
 149       cmd = self.GYPCmd('<path>', vals['gyp_defines'])
 150     else:
 151       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 152
 153     self.PrintCmd(cmd)
 154     return 0
 155
 156   def CmdHelp(self):
 157     if self.args.subcommand:
 158       self.ParseArgs([self.args.subcommand, '--help'])
 159     else:
 160       self.ParseArgs(['--help'])
 161
 162   def CmdValidate(self):
 163     errs = []
 164
 165     # Read the file to make sure it parses.
 166     self.ReadConfigFile()
 167
 168     # Figure out the whole list of configs and ensure that no config is
 169     # listed in more than one category.
 170     all_configs = {}
 171     for config in self.common_dev_configs:
 172       all_configs[config] = 'common_dev_configs'
 173     for config in self.private_configs:
 174       if config in all_configs:
 175         errs.append('config "%s" listed in "private_configs" also '
 176                     'listed in "%s"' % (config, all_configs['config']))
 177       else:
 178         all_configs[config] = 'private_configs'
 179     for config in self.unsupported_configs:
 180       if config in all_configs:
 181         errs.append('config "%s" listed in "unsupported_configs" also '
 182                     'listed in "%s"' % (config, all_configs['config']))
 183       else:
 184         all_configs[config] = 'unsupported_configs'
 185
 186     for master in self.masters:
 187       for builder in self.masters[master]:
 188         config = self.masters[master][builder]
 189         if config in all_configs and all_configs[config] not in self.masters:
 190           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 191                       (config, all_configs[config]))
 192         else:
 193           all_configs[config] = master
 194
 195     # Check that every referenced config actually exists.
 196     for config, loc in all_configs.items():
 197       if not config in self.configs:
 198         errs.append('Unknown config "%s" referenced from "%s".' %
 199                     (config, loc))
 200
 201     # Check that every actual config is actually referenced.
 202     for config in self.configs:
 203       if not config in all_configs:
 204         errs.append('Unused config "%s".' % config)
 205
 206     # Figure out the whole list of mixins, and check that every mixin
 207     # listed by a config or another mixin actually exists.
 208     referenced_mixins = set()
 209     for config, mixins in self.configs.items():
 210       for mixin in mixins:
 211         if not mixin in self.mixins:
 212           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 213                       (mixin, config))
 214         referenced_mixins.add(mixin)
 215
 216     for mixin in self.mixins:
 217       for sub_mixin in self.mixins[mixin].get('mixins', []):
 218         if not sub_mixin in self.mixins:
 219           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 220                       (sub_mixin, mixin))
 221         referenced_mixins.add(sub_mixin)
 222
 223     # Check that every mixin defined is actually referenced somewhere.
 224     for mixin in self.mixins:
 225       if not mixin in referenced_mixins:
 226         errs.append('Unreferenced mixin "%s".' % mixin)
 227
 228     if errs:
 229       raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
 230                     '\n  ' + '\n  '.join(errs))
 231
 232     self.Print('mb config file %s looks ok.' % self.args.config_file)
 233     return 0
 234
 235   def GetConfig(self):
 236     self.ReadConfigFile()
 237     config = self.ConfigFromArgs()
 238     if not config in self.configs:
 239       raise MBErr('Config "%s" not found in %s' %
 240                   (config, self.args.config_file))
 241
 242     return self.FlattenConfig(config)
 243
 244   def ReadConfigFile(self):
 245     if not self.Exists(self.args.config_file):
 246       raise MBErr('config file not found at %s' % self.args.config_file)
 247
 248     try:
 249       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 250     except SyntaxError as e:
 251       raise MBErr('Failed to parse config file "%s": %s' %
 252                  (self.args.config_file, e))
 253
 254     self.common_dev_configs = contents['common_dev_configs']
 255     self.configs = contents['configs']
 256     self.masters = contents['masters']
 257     self.mixins = contents['mixins']
 258     self.private_configs = contents['private_configs']
 259     self.unsupported_configs = contents['unsupported_configs']
 260
 261   def ConfigFromArgs(self):
 262     if self.args.config:
 263       if self.args.master or self.args.builder:
 264         raise MBErr('Can not specific both -c/--config and -m/--master or '
 265                     '-b/--builder')
 266
 267       return self.args.config
 268
 269     if not self.args.master or not self.args.builder:
 270       raise MBErr('Must specify either -c/--config or '
 271                   '(-m/--master and -b/--builder)')
 272
 273     if not self.args.master in self.masters:
 274       raise MBErr('Master name "%s" not found in "%s"' %
 275                   (self.args.master, self.args.config_file))
 276
 277     if not self.args.builder in self.masters[self.args.master]:
 278       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 279                   (self.args.builder, self.args.master, self.args.config_file))
 280
 281     return self.masters[self.args.master][self.args.builder]
 282
 283   def FlattenConfig(self, config):
 284     mixins = self.configs[config]
 285     vals = {
 286       'type': None,
 287       'gn_args': [],
 288       'gyp_defines': '',
 289       'gyp_crosscompile': False,
 290     }
 291
 292     visited = []
 293     self.FlattenMixins(mixins, vals, visited)
 294     return vals
 295
 296   def FlattenMixins(self, mixins, vals, visited):
 297     for m in mixins:
 298       if m not in self.mixins:
 299         raise MBErr('Unknown mixin "%s"' % m)
 300
 301       # TODO: check for cycles in mixins.
 302
 303       visited.append(m)
 304
 305       mixin_vals = self.mixins[m]
 306       if 'type' in mixin_vals:
 307         vals['type'] = mixin_vals['type']
 308       if 'gn_args' in mixin_vals:
 309         if vals['gn_args']:
 310           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 311         else:
 312           vals['gn_args'] = mixin_vals['gn_args']
 313       if 'gyp_crosscompile' in mixin_vals:
 314         vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
 315       if 'gyp_defines' in mixin_vals:
 316         if vals['gyp_defines']:
 317           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 318         else:
 319           vals['gyp_defines'] = mixin_vals['gyp_defines']
 320       if 'mixins' in mixin_vals:
 321         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 322     return vals
 323
 324   def ClobberIfNeeded(self, vals):
 325     path = self.args.path[0]
 326     build_dir = self.ToAbsPath(path)
 327     mb_type_path = self.PathJoin(build_dir, 'mb_type')
 328     needs_clobber = False
 329     new_mb_type = vals['type']
 330     if self.Exists(build_dir):
 331       if self.Exists(mb_type_path):
 332         old_mb_type = self.ReadFile(mb_type_path)
 333         if old_mb_type != new_mb_type:
 334           self.Print("Build type mismatch: was %s, will be %s, clobbering %s" %
 335                      (old_mb_type, new_mb_type, path))
 336           needs_clobber = True
 337       else:
 338         # There is no 'mb_type' file in the build directory, so this probably
 339         # means that the prior build(s) were not done through mb, and we
 340         # have no idea if this was a GYP build or a GN build. Clobber it
 341         # to be safe.
 342         self.Print("%s/mb_type missing, clobbering to be safe" % path)
 343         needs_clobber = True
 344
 345     if needs_clobber:
 346       self.RemoveDirectory(build_dir)
 347
 348     self.MaybeMakeDirectory(build_dir)
 349     self.WriteFile(mb_type_path, new_mb_type)
 350
 351   def RunGNGen(self, vals):
 352     path = self.args.path[0]
 353
 354     cmd = self.GNCmd('gen', path, vals['gn_args'])
 355
 356     swarming_targets = []
 357     if self.args.swarming_targets_file:
 358       # We need GN to generate the list of runtime dependencies for
 359       # the compile targets listed (one per line) in the file so
 360       # we can run them via swarming. We use ninja_to_gn.pyl to convert
 361       # the compile targets to the matching GN labels.
 362       contents = self.ReadFile(self.args.swarming_targets_file)
 363       swarming_targets = contents.splitlines()
 364       gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
 365           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
 366       gn_labels = []
 367       for target in swarming_targets:
 368         if not target in gn_isolate_map:
 369           raise MBErr('test target "%s"  not found in %s' %
 370                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
 371         gn_labels.append(gn_isolate_map[target]['label'])
 372
 373       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 374
 375       # Since GN hasn't run yet, the build directory may not even exist.
 376       self.MaybeMakeDirectory(self.ToAbsPath(path))
 377
 378       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
 379       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 380
 381     ret, _, _ = self.Run(cmd)
 382     if ret:
 383         # If `gn gen` failed, we should exit early rather than trying to
 384         # generate isolates. Run() will have already logged any error output.
 385         self.Print('GN gen failed: %d' % ret)
 386         return ret
 387
 388     for target in swarming_targets:
 389       if gn_isolate_map[target]['type'] == 'gpu_browser_test':
 390         runtime_deps_target = 'browser_tests'
 391       elif gn_isolate_map[target]['type'] == 'script':
 392         # For script targets, the build target is usually a group,
 393         # for which gn generates the runtime_deps next to the stamp file
 394         # for the label, which lives under the obj/ directory.
 395         label = gn_isolate_map[target]['label']
 396         runtime_deps_target = 'obj/%s.stamp' % label.replace(':', '/')
 397       else:
 398         runtime_deps_target = target
 399       if self.platform == 'win32':
 400         deps_path = self.ToAbsPath(path,
 401                                    runtime_deps_target + '.exe.runtime_deps')
 402       else:
 403         deps_path = self.ToAbsPath(path,
 404                                    runtime_deps_target + '.runtime_deps')
 405       if not self.Exists(deps_path):
 406           raise MBErr('did not generate %s' % deps_path)
 407
 408       command, extra_files = self.GetIsolateCommand(target, vals,
 409                                                     gn_isolate_map)
 410
 411       runtime_deps = self.ReadFile(deps_path).splitlines()
 412
 413       isolate_path = self.ToAbsPath(path, target + '.isolate')
 414       self.WriteFile(isolate_path,
 415         pprint.pformat({
 416           'variables': {
 417             'command': command,
 418             'files': sorted(runtime_deps + extra_files),
 419           }
 420         }) + '\n')
 421
 422       self.WriteJSON(
 423         {
 424           'args': [
 425             '--isolated',
 426             self.ToSrcRelPath('%s%s%s.isolated' % (path, self.sep, target)),
 427             '--isolate',
 428             self.ToSrcRelPath('%s%s%s.isolate' % (path, self.sep, target)),
 429           ],
 430           'dir': self.chromium_src_dir,
 431           'version': 1,
 432         },
 433         isolate_path + 'd.gen.json',
 434       )
 435
 436     return ret
 437
 438   def GNCmd(self, subcommand, path, gn_args=''):
 439     if self.platform == 'linux2':
 440       subdir = 'linux64'
 441     elif self.platform == 'darwin':
 442       subdir = 'mac'
 443     else:
 444       subdir = 'win'
 445     gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, 'gn')
 446
 447     cmd = [gn_path, subcommand, path]
 448     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 449     if gn_args:
 450       cmd.append('--args=%s' % gn_args)
 451     return cmd
 452
 453   def RunGYPGen(self, vals):
 454     path = self.args.path[0]
 455
 456     output_dir = self.ParseGYPConfigPath(path)
 457     cmd = self.GYPCmd(output_dir, vals['gyp_defines'])
 458     env = None
 459     if vals['gyp_crosscompile']:
 460       if self.args.verbose:
 461         self.Print('Setting GYP_CROSSCOMPILE=1 in the environment')
 462       env = os.environ.copy()
 463       env['GYP_CROSSCOMPILE'] = '1'
 464     ret, _, _ = self.Run(cmd, env=env)
 465     return ret
 466
 467   def RunGYPAnalyze(self, vals):
 468     output_dir = self.ParseGYPConfigPath(self.args.path[0])
 469     if self.args.verbose:
 470       inp = self.ReadInputJSON(['files', 'targets'])
 471       self.Print()
 472       self.Print('analyze input:')
 473       self.PrintJSON(inp)
 474       self.Print()
 475
 476     cmd = self.GYPCmd(output_dir, vals['gyp_defines'])
 477     cmd.extend(['-f', 'analyzer',
 478                 '-G', 'config_path=%s' % self.args.input_path[0],
 479                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 480     ret, _, _ = self.Run(cmd)
 481     if not ret and self.args.verbose:
 482       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 483       self.Print()
 484       self.Print('analyze output:')
 485       self.PrintJSON(outp)
 486       self.Print()
 487
 488     return ret
 489
 490   def GetIsolateCommand(self, target, vals, gn_isolate_map):
 491     # This needs to mirror the settings in //build/config/ui.gni:
 492     # use_x11 = is_linux && !use_ozone.
 493     # TODO(dpranke): Figure out how to keep this in sync better.
 494     use_x11 = (self.platform == 'linux2' and
 495                not 'target_os="android"' in vals['gn_args'] and
 496                not 'use_ozone=true' in vals['gn_args'])
 497
 498     asan = 'is_asan=true' in vals['gn_args']
 499     msan = 'is_msan=true' in vals['gn_args']
 500     tsan = 'is_tsan=true' in vals['gn_args']
 501
 502     executable_suffix = '.exe' if self.platform == 'win32' else ''
 503
 504     test_type = gn_isolate_map[target]['type']
 505     cmdline = []
 506     extra_files = []
 507
 508     if use_x11 and test_type == 'windowed_test_launcher':
 509       extra_files = [
 510           'xdisplaycheck',
 511           '../../testing/test_env.py',
 512           '../../testing/xvfb.py',
 513       ]
 514       cmdline = [
 515         '../../testing/xvfb.py',
 516         '.',
 517         './' + str(target),
 518         '--brave-new-test-launcher',
 519         '--test-launcher-bot-mode',
 520         '--asan=%d' % asan,
 521         '--msan=%d' % msan,
 522         '--tsan=%d' % tsan,
 523       ]
 524     elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
 525       extra_files = [
 526           '../../testing/test_env.py'
 527       ]
 528       cmdline = [
 529           '../../testing/test_env.py',
 530           './' + str(target) + executable_suffix,
 531           '--brave-new-test-launcher',
 532           '--test-launcher-bot-mode',
 533           '--asan=%d' % asan,
 534           '--msan=%d' % msan,
 535           '--tsan=%d' % tsan,
 536       ]
 537     elif test_type == 'gpu_browser_test':
 538       extra_files = [
 539           '../../testing/test_env.py'
 540       ]
 541       gtest_filter = gn_isolate_map[target]['gtest_filter']
 542       cmdline = [
 543           '../../testing/test_env.py',
 544           './browser_tests' + executable_suffix,
 545           '--test-launcher-bot-mode',
 546           '--enable-gpu',
 547           '--test-launcher-jobs=1',
 548           '--gtest_filter=%s' % gtest_filter,
 549       ]
 550     elif test_type == 'script':
 551       extra_files = [
 552           '../../testing/test_env.py'
 553       ]
 554       cmdline = [
 555           '../../testing/test_env.py',
 556       ] + ['../../' + self.ToSrcRelPath(gn_isolate_map[target]['script'])]
 557     elif test_type in ('raw'):
 558       extra_files = []
 559       cmdline = [
 560           './' + str(target) + executable_suffix,
 561       ] + gn_isolate_map[target].get('args')
 562
 563     else:
 564       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
 565                                 % (target, test_type), output_path=None)
 566
 567     return cmdline, extra_files
 568
 569   def ToAbsPath(self, build_path, *comps):
 570     return self.PathJoin(self.chromium_src_dir,
 571                          self.ToSrcRelPath(build_path),
 572                          *comps)
 573
 574   def ToSrcRelPath(self, path):
 575     """Returns a relative path from the top of the repo."""
 576     # TODO: Support normal paths in addition to source-absolute paths.
 577     assert(path.startswith('//'))
 578     return path[2:].replace('/', self.sep)
 579
 580   def ParseGYPConfigPath(self, path):
 581     rpath = self.ToSrcRelPath(path)
 582     output_dir, _, _ = rpath.rpartition(self.sep)
 583     return output_dir
 584
 585   def GYPCmd(self, output_dir, gyp_defines):
 586     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 587     cmd = [
 588         self.executable,
 589         self.PathJoin('build', 'gyp_chromium'),
 590         '-G',
 591         'output_dir=' + output_dir,
 592     ]
 593     for d in gyp_defines.split(' '):
 594       cmd += ['-D', d]
 595     return cmd
 596
 597   def RunGNAnalyze(self, vals):
 598     # analyze runs before 'gn gen' now, so we need to run gn gen
 599     # in order to ensure that we have a build directory.
 600     ret = self.RunGNGen(vals)
 601     if ret:
 602       return ret
 603
 604     inp = self.ReadInputJSON(['files', 'targets'])
 605     if self.args.verbose:
 606       self.Print()
 607       self.Print('analyze input:')
 608       self.PrintJSON(inp)
 609       self.Print()
 610
 611     output_path = self.args.output_path[0]
 612
 613     # Bail out early if a GN file was modified, since 'gn refs' won't know
 614     # what to do about it.
 615     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 616       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 617       return 0
 618
 619     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 620     if 'all' in inp['targets']:
 621       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 622       return 0
 623
 624     # This shouldn't normally happen, but could due to unusual race conditions,
 625     # like a try job that gets scheduled before a patch lands but runs after
 626     # the patch has landed.
 627     if not inp['files']:
 628       self.Print('Warning: No files modified in patch, bailing out early.')
 629       self.WriteJSON({'targets': [],
 630                       'build_targets': [],
 631                       'status': 'No dependency'}, output_path)
 632       return 0
 633
 634     ret = 0
 635     response_file = self.TempFile()
 636     response_file.write('\n'.join(inp['files']) + '\n')
 637     response_file.close()
 638
 639     matching_targets = []
 640     try:
 641       cmd = self.GNCmd('refs', self.args.path[0]) + [
 642           '@%s' % response_file.name, '--all', '--as=output']
 643       ret, out, _ = self.Run(cmd, force_verbose=False)
 644       if ret and not 'The input matches no targets' in out:
 645         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 646                                   output_path)
 647       build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep
 648       for output in out.splitlines():
 649         build_output = output.replace(build_dir, '')
 650         if build_output in inp['targets']:
 651           matching_targets.append(build_output)
 652
 653       cmd = self.GNCmd('refs', self.args.path[0]) + [
 654           '@%s' % response_file.name, '--all']
 655       ret, out, _ = self.Run(cmd, force_verbose=False)
 656       if ret and not 'The input matches no targets' in out:
 657         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 658                                   output_path)
 659       for label in out.splitlines():
 660         build_target = label[2:]
 661         # We want to accept 'chrome/android:chrome_public_apk' and
 662         # just 'chrome_public_apk'. This may result in too many targets
 663         # getting built, but we can adjust that later if need be.
 664         for input_target in inp['targets']:
 665           if (input_target == build_target or
 666               build_target.endswith(':' + input_target)):
 667             matching_targets.append(input_target)
 668     finally:
 669       self.RemoveFile(response_file.name)
 670
 671     if matching_targets:
 672       # TODO: it could be that a target X might depend on a target Y
 673       # and both would be listed in the input, but we would only need
 674       # to specify target X as a build_target (whereas both X and Y are
 675       # targets). I'm not sure if that optimization is generally worth it.
 676       self.WriteJSON({'targets': sorted(set(matching_targets)),
 677                       'build_targets': sorted(set(matching_targets)),
 678                       'status': 'Found dependency'}, output_path)
 679     else:
 680       self.WriteJSON({'targets': [],
 681                       'build_targets': [],
 682                       'status': 'No dependency'}, output_path)
 683
 684     if self.args.verbose:
 685       outp = json.loads(self.ReadFile(output_path))
 686       self.Print()
 687       self.Print('analyze output:')
 688       self.PrintJSON(outp)
 689       self.Print()
 690
 691     return 0
 692
 693   def ReadInputJSON(self, required_keys):
 694     path = self.args.input_path[0]
 695     output_path = self.args.output_path[0]
 696     if not self.Exists(path):
 697       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 698
 699     try:
 700       inp = json.loads(self.ReadFile(path))
 701     except Exception as e:
 702       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 703                                 (path, e), output_path)
 704
 705     for k in required_keys:
 706       if not k in inp:
 707         self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
 708                                   output_path)
 709
 710     return inp
 711
 712   def WriteFailureAndRaise(self, msg, output_path):
 713     if output_path:
 714       self.WriteJSON({'error': msg}, output_path, force_verbose=True)
 715     raise MBErr(msg)
 716
 717   def WriteJSON(self, obj, path, force_verbose=False):
 718     try:
 719       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
 720                      force_verbose=force_verbose)
 721     except Exception as e:
 722       raise MBErr('Error %s writing to the output path "%s"' %
 723                  (e, path))
 724
 725   def PrintCmd(self, cmd):
 726     if cmd[0] == self.executable:
 727       cmd = ['python'] + cmd[1:]
 728     self.Print(*cmd)
 729
 730   def PrintJSON(self, obj):
 731     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 732
 733   def Print(self, *args, **kwargs):
 734     # This function largely exists so it can be overridden for testing.
 735     print(*args, **kwargs)
 736
 737   def Run(self, cmd, env=None, force_verbose=True):
 738     # This function largely exists so it can be overridden for testing.
 739     if self.args.dryrun or self.args.verbose or force_verbose:
 740       self.PrintCmd(cmd)
 741     if self.args.dryrun:
 742       return 0, '', ''
 743
 744     ret, out, err = self.Call(cmd, env=env)
 745     if self.args.verbose or force_verbose:
 746       if out:
 747         self.Print(out, end='')
 748       if err:
 749         self.Print(err, end='', file=sys.stderr)
 750     return ret, out, err
 751
 752   def Call(self, cmd, env=None):
 753     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 754                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 755                          env=env)
 756     out, err = p.communicate()
 757     return p.returncode, out, err
 758
 759   def ExpandUser(self, path):
 760     # This function largely exists so it can be overridden for testing.
 761     return os.path.expanduser(path)
 762
 763   def Exists(self, path):
 764     # This function largely exists so it can be overridden for testing.
 765     return os.path.exists(path)
 766
 767   def MaybeMakeDirectory(self, path):
 768     try:
 769       os.makedirs(path)
 770     except OSError, e:
 771       if e.errno != errno.EEXIST:
 772         raise
 773
 774   def PathJoin(self, *comps):
 775     # This function largely exists so it can be overriden for testing.
 776     return os.path.join(*comps)
 777
 778   def ReadFile(self, path):
 779     # This function largely exists so it can be overriden for testing.
 780     with open(path) as fp:
 781       return fp.read()
 782
 783   def RemoveFile(self, path):
 784     # This function largely exists so it can be overriden for testing.
 785     os.remove(path)
 786
 787   def RemoveDirectory(self, abs_path):
 788     if self.platform == 'win32':
 789       # In other places in chromium, we often have to retry this command
 790       # because we're worried about other processes still holding on to
 791       # file handles, but when MB is invoked, it will be early enough in the
 792       # build that their should be no other processes to interfere. We
 793       # can change this if need be.
 794       self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
 795     else:
 796       shutil.rmtree(abs_path, ignore_errors=True)
 797
 798   def TempFile(self, mode='w'):
 799     # This function largely exists so it can be overriden for testing.
 800     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 801
 802   def WriteFile(self, path, contents, force_verbose=False):
 803     # This function largely exists so it can be overriden for testing.
 804     if self.args.dryrun or self.args.verbose or force_verbose:
 805       self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
 806     with open(path, 'w') as fp:
 807       return fp.write(contents)
 808
 809
 810 class MBErr(Exception):
 811   pass
 812
 813
 814 if __name__ == '__main__':
 815   try:
 816     sys.exit(main(sys.argv[1:]))
 817   except MBErr as e:
 818     print(e)
 819     sys.exit(1)
 820   except KeyboardInterrupt:
 821     print("interrupted, exiting", stream=sys.stderr)
 822     sys.exit(130)