tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import errno
  17 import json
  18 import os
  19 import pipes
  20 import pprint
  21 import shlex
  22 import shutil
  23 import sys
  24 import subprocess
  25 import tempfile
  26
  27 def main(args):
  28   mbw = MetaBuildWrapper()
  29   mbw.ParseArgs(args)
  30   return mbw.args.func()
  31
  32
  33 class MetaBuildWrapper(object):
  34   def __init__(self):
  35     p = os.path
  36     d = os.path.dirname
  37     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  38     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  39                                  'mb_config.pyl')
  40     self.platform = sys.platform
  41     self.args = argparse.Namespace()
  42     self.configs = {}
  43     self.masters = {}
  44     self.mixins = {}
  45     self.private_configs = []
  46     self.common_dev_configs = []
  47     self.unsupported_configs = []
  48
  49   def ParseArgs(self, argv):
  50     def AddCommonOptions(subp):
  51       subp.add_argument('-b', '--builder',
  52                         help='builder name to look up config from')
  53       subp.add_argument('-m', '--master',
  54                         help='master name to look up config from')
  55       subp.add_argument('-c', '--config',
  56                         help='configuration to analyze')
  57       subp.add_argument('-f', '--config-file', metavar='PATH',
  58                         default=self.default_config,
  59                         help='path to config file '
  60                             '(default is //tools/mb/mb_config.pyl)')
  61       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  62                         help='path to goma directory (default is %(default)s).')
  63       subp.add_argument('-n', '--dryrun', action='store_true',
  64                         help='Do a dry run (i.e., do nothing, just print '
  65                              'the commands that will run)')
  66       subp.add_argument('-q', '--quiet', action='store_true',
  67                         help='Do not print anything on success, '
  68                              'just return an exit code.')
  69       subp.add_argument('-v', '--verbose', action='count',
  70                         help='verbose logging (may specify multiple times).')
  71
  72     parser = argparse.ArgumentParser(prog='mb')
  73     subps = parser.add_subparsers()
  74
  75     subp = subps.add_parser('analyze',
  76                             help='analyze whether changes to a set of files '
  77                                  'will cause a set of binaries to be rebuilt.')
  78     AddCommonOptions(subp)
  79     subp.add_argument('--swarming-targets-file',
  80                       help='save runtime dependencies for targets listed '
  81                            'in file.')
  82     subp.add_argument('path', nargs=1,
  83                       help='path build was generated into.')
  84     subp.add_argument('input_path', nargs=1,
  85                       help='path to a file containing the input arguments '
  86                            'as a JSON object.')
  87     subp.add_argument('output_path', nargs=1,
  88                       help='path to a file containing the output arguments '
  89                            'as a JSON object.')
  90     subp.set_defaults(func=self.CmdAnalyze)
  91
  92     subp = subps.add_parser('gen',
  93                             help='generate a new set of build files')
  94     AddCommonOptions(subp)
  95     subp.add_argument('--swarming-targets-file',
  96                       help='save runtime dependencies for targets listed '
  97                            'in file.')
  98     subp.add_argument('path', nargs=1,
  99                       help='path to generate build into')
 100     subp.set_defaults(func=self.CmdGen)
 101
 102     subp = subps.add_parser('lookup',
 103                             help='look up the command for a given config or '
 104                                  'builder')
 105     AddCommonOptions(subp)
 106     subp.set_defaults(func=self.CmdLookup)
 107
 108     subp = subps.add_parser('validate',
 109                             help='validate the config file')
 110     subp.add_argument('-f', '--config-file', metavar='PATH',
 111                       default=self.default_config,
 112                       help='path to config file '
 113                           '(default is //tools/mb/mb_config.pyl)')
 114     subp.add_argument('-q', '--quiet', action='store_true',
 115                       help='Do not print anything on success, '
 116                            'just return an exit code.')
 117     subp.set_defaults(func=self.CmdValidate)
 118
 119     subp = subps.add_parser('help',
 120                             help='Get help on a subcommand.')
 121     subp.add_argument(nargs='?', action='store', dest='subcommand',
 122                       help='The command to get help for.')
 123     subp.set_defaults(func=self.CmdHelp)
 124
 125     self.args = parser.parse_args(argv)
 126
 127   def CmdAnalyze(self):
 128     vals = self.GetConfig()
 129     if vals['type'] == 'gn':
 130       return self.RunGNAnalyze(vals)
 131     elif vals['type'] == 'gyp':
 132       return self.RunGYPAnalyze(vals)
 133     else:
 134       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 135
 136   def CmdGen(self):
 137     vals = self.GetConfig()
 138     if vals['type'] == 'gn':
 139       return self.RunGNGen(vals)
 140     if vals['type'] == 'gyp':
 141       return self.RunGYPGen(vals)
 142
 143     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 144
 145   def CmdLookup(self):
 146     vals = self.GetConfig()
 147     if vals['type'] == 'gn':
 148       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 149     elif vals['type'] == 'gyp':
 150       if vals['gyp_crosscompile']:
 151         self.Print('GYP_CROSSCOMPILE=1')
 152       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
 153     else:
 154       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 155
 156     self.PrintCmd(cmd)
 157     return 0
 158
 159   def CmdHelp(self):
 160     if self.args.subcommand:
 161       self.ParseArgs([self.args.subcommand, '--help'])
 162     else:
 163       self.ParseArgs(['--help'])
 164
 165   def CmdValidate(self):
 166     errs = []
 167
 168     # Read the file to make sure it parses.
 169     self.ReadConfigFile()
 170
 171     # Figure out the whole list of configs and ensure that no config is
 172     # listed in more than one category.
 173     all_configs = {}
 174     for config in self.common_dev_configs:
 175       all_configs[config] = 'common_dev_configs'
 176     for config in self.private_configs:
 177       if config in all_configs:
 178         errs.append('config "%s" listed in "private_configs" also '
 179                     'listed in "%s"' % (config, all_configs['config']))
 180       else:
 181         all_configs[config] = 'private_configs'
 182     for config in self.unsupported_configs:
 183       if config in all_configs:
 184         errs.append('config "%s" listed in "unsupported_configs" also '
 185                     'listed in "%s"' % (config, all_configs['config']))
 186       else:
 187         all_configs[config] = 'unsupported_configs'
 188
 189     for master in self.masters:
 190       for builder in self.masters[master]:
 191         config = self.masters[master][builder]
 192         if config in all_configs and all_configs[config] not in self.masters:
 193           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 194                       (config, all_configs[config]))
 195         else:
 196           all_configs[config] = master
 197
 198     # Check that every referenced config actually exists.
 199     for config, loc in all_configs.items():
 200       if not config in self.configs:
 201         errs.append('Unknown config "%s" referenced from "%s".' %
 202                     (config, loc))
 203
 204     # Check that every actual config is actually referenced.
 205     for config in self.configs:
 206       if not config in all_configs:
 207         errs.append('Unused config "%s".' % config)
 208
 209     # Figure out the whole list of mixins, and check that every mixin
 210     # listed by a config or another mixin actually exists.
 211     referenced_mixins = set()
 212     for config, mixins in self.configs.items():
 213       for mixin in mixins:
 214         if not mixin in self.mixins:
 215           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 216                       (mixin, config))
 217         referenced_mixins.add(mixin)
 218
 219     for mixin in self.mixins:
 220       for sub_mixin in self.mixins[mixin].get('mixins', []):
 221         if not sub_mixin in self.mixins:
 222           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 223                       (sub_mixin, mixin))
 224         referenced_mixins.add(sub_mixin)
 225
 226     # Check that every mixin defined is actually referenced somewhere.
 227     for mixin in self.mixins:
 228       if not mixin in referenced_mixins:
 229         errs.append('Unreferenced mixin "%s".' % mixin)
 230
 231     if errs:
 232       raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
 233                     '\n  ' + '\n  '.join(errs))
 234
 235     if not self.args.quiet:
 236       self.Print('mb config file %s looks ok.' % self.args.config_file)
 237     return 0
 238
 239   def GetConfig(self):
 240     self.ReadConfigFile()
 241     config = self.ConfigFromArgs()
 242     if not config in self.configs:
 243       raise MBErr('Config "%s" not found in %s' %
 244                   (config, self.args.config_file))
 245
 246     return self.FlattenConfig(config)
 247
 248   def ReadConfigFile(self):
 249     if not self.Exists(self.args.config_file):
 250       raise MBErr('config file not found at %s' % self.args.config_file)
 251
 252     try:
 253       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 254     except SyntaxError as e:
 255       raise MBErr('Failed to parse config file "%s": %s' %
 256                  (self.args.config_file, e))
 257
 258     self.common_dev_configs = contents['common_dev_configs']
 259     self.configs = contents['configs']
 260     self.masters = contents['masters']
 261     self.mixins = contents['mixins']
 262     self.private_configs = contents['private_configs']
 263     self.unsupported_configs = contents['unsupported_configs']
 264
 265   def ConfigFromArgs(self):
 266     if self.args.config:
 267       if self.args.master or self.args.builder:
 268         raise MBErr('Can not specific both -c/--config and -m/--master or '
 269                     '-b/--builder')
 270
 271       return self.args.config
 272
 273     if not self.args.master or not self.args.builder:
 274       raise MBErr('Must specify either -c/--config or '
 275                   '(-m/--master and -b/--builder)')
 276
 277     if not self.args.master in self.masters:
 278       raise MBErr('Master name "%s" not found in "%s"' %
 279                   (self.args.master, self.args.config_file))
 280
 281     if not self.args.builder in self.masters[self.args.master]:
 282       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 283                   (self.args.builder, self.args.master, self.args.config_file))
 284
 285     return self.masters[self.args.master][self.args.builder]
 286
 287   def FlattenConfig(self, config):
 288     mixins = self.configs[config]
 289     vals = {
 290       'type': None,
 291       'gn_args': [],
 292       'gyp_config': [],
 293       'gyp_defines': [],
 294       'gyp_crosscompile': False,
 295     }
 296
 297     visited = []
 298     self.FlattenMixins(mixins, vals, visited)
 299     return vals
 300
 301   def FlattenMixins(self, mixins, vals, visited):
 302     for m in mixins:
 303       if m not in self.mixins:
 304         raise MBErr('Unknown mixin "%s"' % m)
 305
 306       # TODO: check for cycles in mixins.
 307
 308       visited.append(m)
 309
 310       mixin_vals = self.mixins[m]
 311       if 'type' in mixin_vals:
 312         vals['type'] = mixin_vals['type']
 313       if 'gn_args' in mixin_vals:
 314         if vals['gn_args']:
 315           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 316         else:
 317           vals['gn_args'] = mixin_vals['gn_args']
 318       if 'gyp_config' in mixin_vals:
 319         vals['gyp_config'] = mixin_vals['gyp_config']
 320       if 'gyp_crosscompile' in mixin_vals:
 321         vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
 322       if 'gyp_defines' in mixin_vals:
 323         if vals['gyp_defines']:
 324           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 325         else:
 326           vals['gyp_defines'] = mixin_vals['gyp_defines']
 327       if 'mixins' in mixin_vals:
 328         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 329     return vals
 330
 331   def RunGNGen(self, vals):
 332     path = self.args.path[0]
 333
 334     cmd = self.GNCmd('gen', path, vals['gn_args'])
 335
 336     swarming_targets = []
 337     if self.args.swarming_targets_file:
 338       # We need GN to generate the list of runtime dependencies for
 339       # the compile targets listed (one per line) in the file so
 340       # we can run them via swarming. We use ninja_to_gn.pyl to convert
 341       # the compile targets to the matching GN labels.
 342       contents = self.ReadFile(self.args.swarming_targets_file)
 343       swarming_targets = contents.splitlines()
 344       gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
 345           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
 346       gn_labels = []
 347       for target in swarming_targets:
 348         if not target in gn_isolate_map:
 349           raise MBErr('test target "%s"  not found in %s' %
 350                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
 351         gn_labels.append(gn_isolate_map[target]['label'])
 352
 353       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 354
 355       # Since GN hasn't run yet, the build directory may not even exist.
 356       self.MaybeMakeDirectory(self.ToAbsPath(path))
 357
 358       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
 359       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 360
 361     ret, _, _ = self.Run(cmd)
 362
 363     for target in swarming_targets:
 364       if gn_isolate_map[target]['type'] == 'gpu_browser_test':
 365         runtime_deps_target = 'browser_tests'
 366       elif gn_isolate_map[target]['type'] == 'script':
 367         # For script targets, the build target is usually a group,
 368         # for which gn generates the runtime_deps next to the stamp file
 369         # for the label, which lives under the obj/ directory.
 370         label = gn_isolate_map[target]['label']
 371         runtime_deps_target = 'obj/%s.stamp' % label.replace(':', '/')
 372       else:
 373         runtime_deps_target = target
 374       if sys.platform == 'win32':
 375         deps_path = self.ToAbsPath(path,
 376                                    runtime_deps_target + '.exe.runtime_deps')
 377       else:
 378         deps_path = self.ToAbsPath(path,
 379                                    runtime_deps_target + '.runtime_deps')
 380       if not self.Exists(deps_path):
 381           raise MBErr('did not generate %s' % deps_path)
 382
 383       command, extra_files = self.GetIsolateCommand(target, vals,
 384                                                     gn_isolate_map)
 385
 386       runtime_deps = self.ReadFile(deps_path).splitlines()
 387
 388       isolate_path = self.ToAbsPath(path, target + '.isolate')
 389       self.WriteFile(isolate_path,
 390         pprint.pformat({
 391           'variables': {
 392             'command': command,
 393             'files': sorted(runtime_deps + extra_files),
 394           }
 395         }) + '\n')
 396
 397       self.WriteJSON(
 398         {
 399           'args': [
 400             '--isolated',
 401             self.ToSrcRelPath('%s%s%s.isolated' % (path, os.sep, target)),
 402             '--isolate',
 403             self.ToSrcRelPath('%s%s%s.isolate' % (path, os.sep, target)),
 404           ],
 405           'dir': self.chromium_src_dir,
 406           'version': 1,
 407         },
 408         isolate_path + 'd.gen.json',
 409       )
 410
 411
 412     return ret
 413
 414   def GNCmd(self, subcommand, path, gn_args=''):
 415     if self.platform == 'linux2':
 416       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
 417                              'gn')
 418     elif self.platform == 'darwin':
 419       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
 420                              'gn')
 421     else:
 422       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
 423                              'gn.exe')
 424
 425     cmd = [gn_path, subcommand, path]
 426     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 427     if gn_args:
 428       cmd.append('--args=%s' % gn_args)
 429     return cmd
 430
 431   def RunGYPGen(self, vals):
 432     path = self.args.path[0]
 433
 434     output_dir, gyp_config = self.ParseGYPConfigPath(path)
 435     if gyp_config != vals['gyp_config']:
 436       raise MBErr('The last component of the path (%s) must match the '
 437                   'GYP configuration specified in the config (%s), and '
 438                   'it does not.' % (gyp_config, vals['gyp_config']))
 439     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 440     env = None
 441     if vals['gyp_crosscompile']:
 442       if self.args.verbose:
 443         self.Print('Setting GYP_CROSSCOMPILE=1 in the environment')
 444       env = os.environ.copy()
 445       env['GYP_CROSSCOMPILE'] = '1'
 446     ret, _, _ = self.Run(cmd, env=env)
 447     return ret
 448
 449   def RunGYPAnalyze(self, vals):
 450     output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
 451     if gyp_config != vals['gyp_config']:
 452       raise MBErr('The last component of the path (%s) must match the '
 453                   'GYP configuration specified in the config (%s), and '
 454                   'it does not.' % (gyp_config, vals['gyp_config']))
 455     if self.args.verbose:
 456       inp = self.ReadInputJSON(['files', 'targets'])
 457       self.Print()
 458       self.Print('analyze input:')
 459       self.PrintJSON(inp)
 460       self.Print()
 461
 462     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 463     cmd.extend(['-f', 'analyzer',
 464                 '-G', 'config_path=%s' % self.args.input_path[0],
 465                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 466     ret, _, _ = self.Run(cmd)
 467     if not ret and self.args.verbose:
 468       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 469       self.Print()
 470       self.Print('analyze output:')
 471       self.PrintJSON(outp)
 472       self.Print()
 473
 474     return ret
 475
 476   def RunGNIsolate(self, vals):
 477     build_path = self.args.path[0]
 478     inp = self.ReadInputJSON(['targets'])
 479     if self.args.verbose:
 480       self.Print()
 481       self.Print('isolate input:')
 482       self.PrintJSON(inp)
 483       self.Print()
 484     output_path = self.args.output_path[0]
 485
 486     for target in inp['targets']:
 487       runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
 488
 489       if not self.Exists(runtime_deps_path):
 490         self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
 491                                   output_path)
 492
 493       command, extra_files = self.GetIsolateCommand(target, vals, None)
 494
 495       runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
 496
 497
 498       isolate_path = self.ToAbsPath(build_path, target + '.isolate')
 499       self.WriteFile(isolate_path,
 500         pprint.pformat({
 501           'variables': {
 502             'command': command,
 503             'files': sorted(runtime_deps + extra_files),
 504           }
 505         }) + '\n')
 506
 507       self.WriteJSON(
 508         {
 509           'args': [
 510             '--isolated',
 511             self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
 512             '--isolate',
 513             self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
 514           ],
 515           'dir': self.chromium_src_dir,
 516           'version': 1,
 517         },
 518         isolate_path + 'd.gen.json',
 519       )
 520
 521     return 0
 522
 523   def GetIsolateCommand(self, target, vals, gn_isolate_map):
 524     # This needs to mirror the settings in //build/config/ui.gni:
 525     # use_x11 = is_linux && !use_ozone.
 526     # TODO(dpranke): Figure out how to keep this in sync better.
 527     use_x11 = (sys.platform == 'linux2' and
 528                not 'target_os="android"' in vals['gn_args'] and
 529                not 'use_ozone=true' in vals['gn_args'])
 530
 531     asan = 'is_asan=true' in vals['gn_args']
 532     msan = 'is_msan=true' in vals['gn_args']
 533     tsan = 'is_tsan=true' in vals['gn_args']
 534
 535     executable_suffix = '.exe' if sys.platform == 'win32' else ''
 536
 537     test_type = gn_isolate_map[target]['type']
 538     cmdline = []
 539     extra_files = []
 540
 541     if use_x11 and test_type == 'windowed_test_launcher':
 542       extra_files = [
 543           'xdisplaycheck',
 544           '../../testing/test_env.py',
 545           '../../testing/xvfb.py',
 546       ]
 547       cmdline = [
 548         '../../testing/xvfb.py',
 549         '.',
 550         './' + str(target),
 551         '--brave-new-test-launcher',
 552         '--test-launcher-bot-mode',
 553         '--asan=%d' % asan,
 554         '--msan=%d' % msan,
 555         '--tsan=%d' % tsan,
 556       ]
 557     elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
 558       extra_files = [
 559           '../../testing/test_env.py'
 560       ]
 561       cmdline = [
 562           '../../testing/test_env.py',
 563           './' + str(target) + executable_suffix,
 564           '--brave-new-test-launcher',
 565           '--test-launcher-bot-mode',
 566           '--asan=%d' % asan,
 567           '--msan=%d' % msan,
 568           '--tsan=%d' % tsan,
 569       ]
 570     elif test_type == 'gpu_browser_test':
 571       extra_files = [
 572           '../../testing/test_env.py'
 573       ]
 574       gtest_filter = gn_isolate_map[target]['gtest_filter']
 575       cmdline = [
 576           '../../testing/test_env.py',
 577           './browser_tests' + executable_suffix,
 578           '--test-launcher-bot-mode',
 579           '--enable-gpu',
 580           '--test-launcher-jobs=1',
 581           '--gtest_filter=%s' % gtest_filter,
 582       ]
 583     elif test_type == 'script':
 584       extra_files = [
 585           '../../testing/test_env.py'
 586       ]
 587       cmdline = [
 588           '../../testing/test_env.py',
 589       ] + ['../../' + self.ToSrcRelPath(gn_isolate_map[target]['script'])]
 590     elif test_type in ('raw'):
 591       extra_files = []
 592       cmdline = [
 593           './' + str(target) + executable_suffix,
 594       ] + gn_isolate_map[target].get('args')
 595
 596     else:
 597       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
 598                                 % (target, test_type), output_path=None)
 599
 600     return cmdline, extra_files
 601
 602   def ToAbsPath(self, build_path, *comps):
 603     return os.path.join(self.chromium_src_dir,
 604                         self.ToSrcRelPath(build_path),
 605                         *comps)
 606
 607   def ToSrcRelPath(self, path):
 608     """Returns a relative path from the top of the repo."""
 609     # TODO: Support normal paths in addition to source-absolute paths.
 610     assert(path.startswith('//'))
 611     return path[2:].replace('/', os.sep)
 612
 613   def ParseGYPConfigPath(self, path):
 614     rpath = self.ToSrcRelPath(path)
 615     output_dir, _, config = rpath.rpartition('/')
 616     self.CheckGYPConfigIsSupported(config, path)
 617     return output_dir, config
 618
 619   def CheckGYPConfigIsSupported(self, config, path):
 620     if config not in ('Debug', 'Release'):
 621       if (sys.platform in ('win32', 'cygwin') and
 622           config not in ('Debug_x64', 'Release_x64')):
 623         raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
 624                     config, path)
 625
 626   def GYPCmd(self, output_dir, gyp_defines, config):
 627     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 628     cmd = [
 629         sys.executable,
 630         os.path.join('build', 'gyp_chromium'),
 631         '-G',
 632         'output_dir=' + output_dir,
 633         '-G',
 634         'config=' + config,
 635     ]
 636     for d in shlex.split(gyp_defines):
 637       cmd += ['-D', d]
 638     return cmd
 639
 640   def RunGNAnalyze(self, vals):
 641     # analyze runs before 'gn gen' now, so we need to run gn gen
 642     # in order to ensure that we have a build directory.
 643     ret = self.RunGNGen(vals)
 644     if ret:
 645       return ret
 646
 647     inp = self.ReadInputJSON(['files', 'targets'])
 648     if self.args.verbose:
 649       self.Print()
 650       self.Print('analyze input:')
 651       self.PrintJSON(inp)
 652       self.Print()
 653
 654     output_path = self.args.output_path[0]
 655
 656     # Bail out early if a GN file was modified, since 'gn refs' won't know
 657     # what to do about it.
 658     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 659       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 660       return 0
 661
 662     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 663     if 'all' in inp['targets']:
 664       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 665       return 0
 666
 667     # This shouldn't normally happen, but could due to unusual race conditions,
 668     # like a try job that gets scheduled before a patch lands but runs after
 669     # the patch has landed.
 670     if not inp['files']:
 671       self.Print('Warning: No files modified in patch, bailing out early.')
 672       self.WriteJSON({'targets': [],
 673                       'build_targets': [],
 674                       'status': 'No dependency'}, output_path)
 675       return 0
 676
 677     ret = 0
 678     response_file = self.TempFile()
 679     response_file.write('\n'.join(inp['files']) + '\n')
 680     response_file.close()
 681
 682     matching_targets = []
 683     try:
 684       cmd = self.GNCmd('refs', self.args.path[0]) + [
 685           '@%s' % response_file.name, '--all', '--as=output']
 686       ret, out, _ = self.Run(cmd)
 687       if ret and not 'The input matches no targets' in out:
 688         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 689                                   output_path)
 690       build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
 691       for output in out.splitlines():
 692         build_output = output.replace(build_dir, '')
 693         if build_output in inp['targets']:
 694           matching_targets.append(build_output)
 695
 696       cmd = self.GNCmd('refs', self.args.path[0]) + [
 697           '@%s' % response_file.name, '--all']
 698       ret, out, _ = self.Run(cmd)
 699       if ret and not 'The input matches no targets' in out:
 700         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 701                                   output_path)
 702       for label in out.splitlines():
 703         build_target = label[2:]
 704         # We want to accept 'chrome/android:chrome_public_apk' and
 705         # just 'chrome_public_apk'. This may result in too many targets
 706         # getting built, but we can adjust that later if need be.
 707         for input_target in inp['targets']:
 708           if (input_target == build_target or
 709               build_target.endswith(':' + input_target)):
 710             matching_targets.append(input_target)
 711     finally:
 712       self.RemoveFile(response_file.name)
 713
 714     if matching_targets:
 715       # TODO: it could be that a target X might depend on a target Y
 716       # and both would be listed in the input, but we would only need
 717       # to specify target X as a build_target (whereas both X and Y are
 718       # targets). I'm not sure if that optimization is generally worth it.
 719       self.WriteJSON({'targets': sorted(matching_targets),
 720                       'build_targets': sorted(matching_targets),
 721                       'status': 'Found dependency'}, output_path)
 722     else:
 723       self.WriteJSON({'targets': [],
 724                       'build_targets': [],
 725                       'status': 'No dependency'}, output_path)
 726
 727     if not ret and self.args.verbose:
 728       outp = json.loads(self.ReadFile(output_path))
 729       self.Print()
 730       self.Print('analyze output:')
 731       self.PrintJSON(outp)
 732       self.Print()
 733
 734     return 0
 735
 736   def ReadInputJSON(self, required_keys):
 737     path = self.args.input_path[0]
 738     output_path = self.args.output_path[0]
 739     if not self.Exists(path):
 740       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 741
 742     try:
 743       inp = json.loads(self.ReadFile(path))
 744     except Exception as e:
 745       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 746                                 (path, e), output_path)
 747
 748     for k in required_keys:
 749       if not k in inp:
 750         self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
 751                                   output_path)
 752
 753     return inp
 754
 755   def WriteFailureAndRaise(self, msg, output_path):
 756     if output_path:
 757       self.WriteJSON({'error': msg}, output_path)
 758     raise MBErr(msg)
 759
 760   def WriteJSON(self, obj, path):
 761     try:
 762       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
 763     except Exception as e:
 764       raise MBErr('Error %s writing to the output path "%s"' %
 765                  (e, path))
 766
 767   def PrintCmd(self, cmd):
 768     if cmd[0] == sys.executable:
 769       cmd = ['python'] + cmd[1:]
 770     self.Print(*[pipes.quote(c) for c in cmd])
 771
 772   def PrintJSON(self, obj):
 773     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 774
 775   def Print(self, *args, **kwargs):
 776     # This function largely exists so it can be overridden for testing.
 777     print(*args, **kwargs)
 778
 779   def Run(self, cmd, env=None):
 780     # This function largely exists so it can be overridden for testing.
 781     if self.args.dryrun or self.args.verbose:
 782       self.PrintCmd(cmd)
 783     if self.args.dryrun:
 784       return 0, '', ''
 785     ret, out, err = self.Call(cmd, env=env)
 786     if self.args.verbose:
 787       if out:
 788         self.Print(out, end='')
 789       if err:
 790         self.Print(err, end='', file=sys.stderr)
 791     return ret, out, err
 792
 793   def Call(self, cmd, env=None):
 794     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 795                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 796                          env=env)
 797     out, err = p.communicate()
 798     return p.returncode, out, err
 799
 800   def ExpandUser(self, path):
 801     # This function largely exists so it can be overridden for testing.
 802     return os.path.expanduser(path)
 803
 804   def Exists(self, path):
 805     # This function largely exists so it can be overridden for testing.
 806     return os.path.exists(path)
 807
 808   def MaybeMakeDirectory(self, path):
 809     try:
 810       os.makedirs(path)
 811     except OSError, e:
 812       if e.errno != errno.EEXIST:
 813         raise
 814
 815   def ReadFile(self, path):
 816     # This function largely exists so it can be overriden for testing.
 817     with open(path) as fp:
 818       return fp.read()
 819
 820   def RemoveFile(self, path):
 821     # This function largely exists so it can be overriden for testing.
 822     os.remove(path)
 823
 824   def TempFile(self, mode='w'):
 825     # This function largely exists so it can be overriden for testing.
 826     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 827
 828   def WriteFile(self, path, contents):
 829     # This function largely exists so it can be overriden for testing.
 830     if self.args.dryrun or self.args.verbose:
 831       self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
 832     with open(path, 'w') as fp:
 833       return fp.write(contents)
 834
 835
 836 class MBErr(Exception):
 837   pass
 838
 839
 840 if __name__ == '__main__':
 841   try:
 842     sys.exit(main(sys.argv[1:]))
 843   except MBErr as e:
 844     print(e)
 845     sys.exit(1)
 846   except KeyboardInterrupt:
 847     print("interrupted, exiting", stream=sys.stderr)
 848     sys.exit(130)