tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import json
  17 import os
  18 import pipes
  19 import shlex
  20 import shutil
  21 import sys
  22 import subprocess
  23 import tempfile
  24
  25
  26 def main(args):
  27   mbw = MetaBuildWrapper()
  28   mbw.ParseArgs(args)
  29   return mbw.args.func()
  30
  31
  32 class MetaBuildWrapper(object):
  33   def __init__(self):
  34     p = os.path
  35     d = os.path.dirname
  36     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  37     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  38                                  'mb_config.pyl')
  39     self.platform = sys.platform
  40     self.args = argparse.Namespace()
  41     self.configs = {}
  42     self.masters = {}
  43     self.mixins = {}
  44     self.private_configs = []
  45     self.common_dev_configs = []
  46     self.unsupported_configs = []
  47
  48   def ParseArgs(self, argv):
  49     def AddCommonOptions(subp):
  50       subp.add_argument('-b', '--builder',
  51                         help='builder name to look up config from')
  52       subp.add_argument('-m', '--master',
  53                         help='master name to look up config from')
  54       subp.add_argument('-c', '--config',
  55                         help='configuration to analyze')
  56       subp.add_argument('-f', '--config-file', metavar='PATH',
  57                         default=self.default_config,
  58                         help='path to config file '
  59                             '(default is //tools/mb/mb_config.pyl)')
  60       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  61                         help='path to goma directory (default is %(default)s).')
  62       subp.add_argument('-n', '--dryrun', action='store_true',
  63                         help='Do a dry run (i.e., do nothing, just print '
  64                              'the commands that will run)')
  65       subp.add_argument('-q', '--quiet', action='store_true',
  66                         help='Do not print anything, just return an exit '
  67                              'code.')
  68       subp.add_argument('-v', '--verbose', action='count',
  69                         help='verbose logging (may specify multiple times).')
  70
  71     parser = argparse.ArgumentParser(prog='mb')
  72     subps = parser.add_subparsers()
  73
  74     subp = subps.add_parser('analyze',
  75                             help='analyze whether changes to a set of files '
  76                                  'will cause a set of binaries to be rebuilt.')
  77     AddCommonOptions(subp)
  78     subp.add_argument('path', type=str, nargs=1,
  79                       help='path build was generated into.')
  80     subp.add_argument('input_path', nargs=1,
  81                       help='path to a file containing the input arguments '
  82                            'as a JSON object.')
  83     subp.add_argument('output_path', nargs=1,
  84                       help='path to a file containing the output arguments '
  85                            'as a JSON object.')
  86     subp.set_defaults(func=self.CmdAnalyze)
  87
  88     subp = subps.add_parser('gen',
  89                             help='generate a new set of build files')
  90     AddCommonOptions(subp)
  91     subp.add_argument('path', type=str, nargs=1,
  92                       help='path to generate build into')
  93     subp.set_defaults(func=self.CmdGen)
  94
  95     subp = subps.add_parser('lookup',
  96                             help='look up the command for a given config or '
  97                                  'builder')
  98     AddCommonOptions(subp)
  99     subp.set_defaults(func=self.CmdLookup)
 100
 101     subp = subps.add_parser('validate',
 102                             help='validate the config file')
 103     AddCommonOptions(subp)
 104     subp.set_defaults(func=self.CmdValidate)
 105
 106     subp = subps.add_parser('help',
 107                             help='Get help on a subcommand.')
 108     subp.add_argument(nargs='?', action='store', dest='subcommand',
 109                       help='The command to get help for.')
 110     subp.set_defaults(func=self.CmdHelp)
 111
 112     self.args = parser.parse_args(argv)
 113
 114   def CmdAnalyze(self):
 115     vals = self.GetConfig()
 116     if vals['type'] == 'gn':
 117       return self.RunGNAnalyze(vals)
 118     elif vals['type'] == 'gyp':
 119       return self.RunGYPAnalyze(vals)
 120     else:
 121       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 122
 123   def CmdGen(self):
 124     vals = self.GetConfig()
 125     if vals['type'] == 'gn':
 126       return self.RunGNGen(self.args.path[0], vals)
 127     if vals['type'] == 'gyp':
 128       return self.RunGYPGen(self.args.path[0], vals)
 129
 130     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 131
 132   def CmdLookup(self):
 133     vals = self.GetConfig()
 134     if vals['type'] == 'gn':
 135       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 136     elif vals['type'] == 'gyp':
 137       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
 138     else:
 139       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 140
 141     self.PrintCmd(cmd)
 142     return 0
 143
 144   def CmdHelp(self):
 145     if self.args.subcommand:
 146       self.ParseArgs([self.args.subcommand, '--help'])
 147     else:
 148       self.ParseArgs(['--help'])
 149
 150   def CmdValidate(self):
 151     errs = []
 152
 153     # Read the file to make sure it parses.
 154     self.ReadConfigFile()
 155
 156     # Figure out the whole list of configs and ensure that no config is
 157     # listed in more than one category.
 158     all_configs = {}
 159     for config in self.common_dev_configs:
 160       all_configs[config] = 'common_dev_configs'
 161     for config in self.private_configs:
 162       if config in all_configs:
 163         errs.append('config "%s" listed in "private_configs" also '
 164                     'listed in "%s"' % (config, all_configs['config']))
 165       else:
 166         all_configs[config] = 'private_configs'
 167     for config in self.unsupported_configs:
 168       if config in all_configs:
 169         errs.append('config "%s" listed in "unsupported_configs" also '
 170                     'listed in "%s"' % (config, all_configs['config']))
 171       else:
 172         all_configs[config] = 'unsupported_configs'
 173
 174     for master in self.masters:
 175       for builder in self.masters[master]:
 176         config = self.masters[master][builder]
 177         if config in all_configs and all_configs[config] not in self.masters:
 178           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 179                       (config, all_configs[config]))
 180         else:
 181           all_configs[config] = master
 182
 183     # Check that every referenced config actually exists.
 184     for config, loc in all_configs.items():
 185       if not config in self.configs:
 186         errs.append('Unknown config "%s" referenced from "%s".' %
 187                     (config, loc))
 188
 189     # Check that every actual config is actually referenced.
 190     for config in self.configs:
 191       if not config in all_configs:
 192         errs.append('Unused config "%s".' % config)
 193
 194     # Figure out the whole list of mixins, and check that every mixin
 195     # listed by a config or another mixin actually exists.
 196     referenced_mixins = set()
 197     for config, mixins in self.configs.items():
 198       for mixin in mixins:
 199         if not mixin in self.mixins:
 200           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 201                       (mixin, config))
 202         referenced_mixins.add(mixin)
 203
 204     for mixin in self.mixins:
 205       for sub_mixin in self.mixins[mixin].get('mixins', []):
 206         if not sub_mixin in self.mixins:
 207           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 208                       (sub_mixin, mixin))
 209         referenced_mixins.add(sub_mixin)
 210
 211     # Check that every mixin defined is actually referenced somewhere.
 212     for mixin in self.mixins:
 213       if not mixin in referenced_mixins:
 214         errs.append('Unreferenced mixin "%s".' % mixin)
 215
 216     if errs:
 217       raise MBErr('mb config file %s has problems:\n  ' + '\n  '.join(errs))
 218
 219     if not self.args.quiet:
 220       self.Print('mb config file %s looks ok.' % self.args.config_file)
 221     return 0
 222
 223   def GetConfig(self):
 224     self.ReadConfigFile()
 225     config = self.ConfigFromArgs()
 226     if not config in self.configs:
 227       raise MBErr('Config "%s" not found in %s' %
 228                   (config, self.args.config_file))
 229
 230     return self.FlattenConfig(config)
 231
 232   def ReadConfigFile(self):
 233     if not self.Exists(self.args.config_file):
 234       raise MBErr('config file not found at %s' % self.args.config_file)
 235
 236     try:
 237       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 238     except SyntaxError as e:
 239       raise MBErr('Failed to parse config file "%s": %s' %
 240                  (self.args.config_file, e))
 241
 242     self.common_dev_configs = contents['common_dev_configs']
 243     self.configs = contents['configs']
 244     self.masters = contents['masters']
 245     self.mixins = contents['mixins']
 246     self.private_configs = contents['private_configs']
 247     self.unsupported_configs = contents['unsupported_configs']
 248
 249   def ConfigFromArgs(self):
 250     if self.args.config:
 251       if self.args.master or self.args.builder:
 252         raise MBErr('Can not specific both -c/--config and -m/--master or '
 253                     '-b/--builder')
 254
 255       return self.args.config
 256
 257     if not self.args.master or not self.args.builder:
 258       raise MBErr('Must specify either -c/--config or '
 259                   '(-m/--master and -b/--builder)')
 260
 261     if not self.args.master in self.masters:
 262       raise MBErr('Master name "%s" not found in "%s"' %
 263                   (self.args.master, self.args.config_file))
 264
 265     if not self.args.builder in self.masters[self.args.master]:
 266       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 267                   (self.args.builder, self.args.master, self.args.config_file))
 268
 269     return self.masters[self.args.master][self.args.builder]
 270
 271   def FlattenConfig(self, config):
 272     mixins = self.configs[config]
 273     vals = {
 274       'type': None,
 275       'gn_args': [],
 276       'gyp_config': [],
 277       'gyp_defines': [],
 278     }
 279
 280     visited = []
 281     self.FlattenMixins(mixins, vals, visited)
 282     return vals
 283
 284   def FlattenMixins(self, mixins, vals, visited):
 285     for m in mixins:
 286       if m not in self.mixins:
 287         raise MBErr('Unknown mixin "%s"' % m)
 288
 289       # TODO: check for cycles in mixins.
 290
 291       visited.append(m)
 292
 293       mixin_vals = self.mixins[m]
 294       if 'type' in mixin_vals:
 295         vals['type'] = mixin_vals['type']
 296       if 'gn_args' in mixin_vals:
 297         if vals['gn_args']:
 298           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 299         else:
 300           vals['gn_args'] = mixin_vals['gn_args']
 301       if 'gyp_config' in mixin_vals:
 302         vals['gyp_config'] = mixin_vals['gyp_config']
 303       if 'gyp_defines' in mixin_vals:
 304         if vals['gyp_defines']:
 305           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 306         else:
 307           vals['gyp_defines'] = mixin_vals['gyp_defines']
 308       if 'mixins' in mixin_vals:
 309         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 310     return vals
 311
 312   def RunGNGen(self, path, vals):
 313     cmd = self.GNCmd('gen', path, vals['gn_args'])
 314     ret, _, _ = self.Run(cmd)
 315     return ret
 316
 317   def GNCmd(self, subcommand, path, gn_args=''):
 318     if self.platform == 'linux2':
 319       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
 320                              'gn')
 321     elif self.platform == 'darwin':
 322       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
 323                              'gn')
 324     else:
 325       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
 326                              'gn.exe')
 327
 328     cmd = [gn_path, subcommand, path]
 329     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 330     if gn_args:
 331       cmd.append('--args=%s' % gn_args)
 332     return cmd
 333
 334   def RunGYPGen(self, path, vals):
 335     output_dir, gyp_config = self.ParseGYPConfigPath(path)
 336     if gyp_config != vals['gyp_config']:
 337       raise MBErr('The last component of the path (%s) must match the '
 338                   'GYP configuration specified in the config (%s), and '
 339                   'it does not.' % (gyp_config, vals['gyp_config']))
 340     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 341     ret, _, _ = self.Run(cmd)
 342     return ret
 343
 344   def RunGYPAnalyze(self, vals):
 345     output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
 346     if gyp_config != vals['gyp_config']:
 347       raise MBErr('The last component of the path (%s) must match the '
 348                   'GYP configuration specified in the config (%s), and '
 349                   'it does not.' % (gyp_config, vals['gyp_config']))
 350     if self.args.verbose:
 351       inp = self.GetAnalyzeInput()
 352       self.Print()
 353       self.Print('analyze input:')
 354       self.PrintJSON(inp)
 355       self.Print()
 356
 357     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 358     cmd.extend(['-G', 'config_path=%s' % self.args.input_path[0],
 359                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 360     ret, _, _ = self.Run(cmd)
 361     if not ret and self.args.verbose:
 362       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 363       self.Print()
 364       self.Print('analyze output:')
 365       self.PrintJSON(inp)
 366       self.Print()
 367
 368     return ret
 369
 370   def ToSrcRelPath(self, path):
 371     """Returns a relative path from the top of the repo."""
 372     # TODO: Support normal paths in addition to source-absolute paths.
 373     assert(path.startswith('//'))
 374     return path[2:]
 375
 376   def ParseGYPConfigPath(self, path):
 377     rpath = self.ToSrcRelPath(path)
 378     output_dir, _, config = rpath.rpartition('/')
 379     self.CheckGYPConfigIsSupported(config, path)
 380     return output_dir, config
 381
 382   def CheckGYPConfigIsSupported(self, config, path):
 383     if config not in ('Debug', 'Release'):
 384       if (sys.platform in ('win32', 'cygwin') and
 385           config not in ('Debug_x64', 'Release_x64')):
 386         raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
 387                     config, path)
 388
 389   def GYPCmd(self, output_dir, gyp_defines, config):
 390     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 391     cmd = [
 392         sys.executable,
 393         os.path.join('build', 'gyp_chromium'),
 394         '-G',
 395         'output_dir=' + output_dir,
 396         '-G',
 397         'config=' + config,
 398     ]
 399     for d in shlex.split(gyp_defines):
 400       cmd += ['-D', d]
 401     return cmd
 402
 403   def RunGNAnalyze(self, _vals):
 404     inp = self.GetAnalyzeInput()
 405     if self.args.verbose:
 406       self.Print()
 407       self.Print('analyze input:')
 408       self.PrintJSON(inp)
 409       self.Print()
 410
 411     output_path = self.args.output_path[0]
 412
 413     # Bail out early if a GN file was modified, since 'gn refs' won't know
 414     # what to do about it.
 415     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 416       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 417       return 0
 418
 419     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 420     if 'all' in inp['targets']:
 421       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 422       return 0
 423
 424     ret = 0
 425     response_file = self.TempFile()
 426     response_file.write('\n'.join(inp['files']) + '\n')
 427     response_file.close()
 428
 429     matching_targets = []
 430     try:
 431       cmd = self.GNCmd('refs', self.args.path[0]) + [
 432           '@%s' % response_file.name, '--all', '--as=output']
 433       ret, out, _ = self.Run(cmd)
 434       if ret and not 'The input matches no targets' in out:
 435         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 436                                   output_path)
 437       build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
 438       for output in out.splitlines():
 439         build_output = output.replace(build_dir, '')
 440         if build_output in inp['targets']:
 441           matching_targets.append(build_output)
 442
 443       cmd = self.GNCmd('refs', self.args.path[0]) + [
 444           '@%s' % response_file.name, '--all']
 445       ret, out, _ = self.Run(cmd)
 446       if ret and not 'The input matches no targets' in out:
 447         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 448                                   output_path)
 449       for label in out.splitlines():
 450         build_target = label[2:]
 451         # We want to accept 'chrome/android:chrome_shell_apk' and
 452         # just 'chrome_shell_apk'. This may result in too many targets
 453         # getting built, but we can adjust that later if need be.
 454         for input_target in inp['targets']:
 455           if (input_target == build_target or
 456               build_target.endswith(':' + input_target)):
 457             matching_targets.append(input_target)
 458     finally:
 459       self.RemoveFile(response_file.name)
 460
 461     if matching_targets:
 462       # TODO: it could be that a target X might depend on a target Y
 463       # and both would be listed in the input, but we would only need
 464       # to specify target X as a build_target (whereas both X and Y are
 465       # targets). I'm not sure if that optimization is generally worth it.
 466       self.WriteJSON({'targets': sorted(matching_targets),
 467                       'build_targets': sorted(matching_targets),
 468                       'status': 'Found dependency'}, output_path)
 469     else:
 470       self.WriteJSON({'targets': [],
 471                       'build_targets': [],
 472                       'status': 'No dependency'}, output_path)
 473
 474     if not ret and self.args.verbose:
 475       outp = json.loads(self.ReadFile(output_path))
 476       self.Print()
 477       self.Print('analyze output:')
 478       self.PrintJSON(outp)
 479       self.Print()
 480
 481     return 0
 482
 483   def GetAnalyzeInput(self):
 484     path = self.args.input_path[0]
 485     output_path = self.args.output_path[0]
 486     if not self.Exists(path):
 487       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 488
 489     try:
 490       inp = json.loads(self.ReadFile(path))
 491     except Exception as e:
 492       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 493                                 (path, e), output_path)
 494     if not 'files' in inp:
 495       self.WriteFailureAndRaise('input file is missing a "files" key',
 496                                 output_path)
 497     if not 'targets' in inp:
 498       self.WriteFailureAndRaise('input file is missing a "targets" key',
 499                                 output_path)
 500
 501     return inp
 502
 503   def WriteFailureAndRaise(self, msg, path):
 504     self.WriteJSON({'error': msg}, path)
 505     raise MBErr(msg)
 506
 507   def WriteJSON(self, obj, path):
 508     try:
 509       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
 510     except Exception as e:
 511       raise MBErr('Error %s writing to the output path "%s"' %
 512                  (e, path))
 513
 514   def PrintCmd(self, cmd):
 515     if cmd[0] == sys.executable:
 516       cmd = ['python'] + cmd[1:]
 517     self.Print(*[pipes.quote(c) for c in cmd])
 518
 519   def PrintJSON(self, obj):
 520     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 521
 522   def Print(self, *args, **kwargs):
 523     # This function largely exists so it can be overridden for testing.
 524     print(*args, **kwargs)
 525
 526   def Run(self, cmd):
 527     # This function largely exists so it can be overridden for testing.
 528     if self.args.dryrun or self.args.verbose:
 529       self.PrintCmd(cmd)
 530     if self.args.dryrun:
 531       return 0, '', ''
 532     ret, out, err = self.Call(cmd)
 533     if self.args.verbose:
 534       if out:
 535         self.Print(out, end='')
 536       if err:
 537         self.Print(err, end='', file=sys.stderr)
 538     return ret, out, err
 539
 540   def Call(self, cmd):
 541     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 542                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 543     out, err = p.communicate()
 544     return p.returncode, out, err
 545
 546   def ExpandUser(self, path):
 547     # This function largely exists so it can be overridden for testing.
 548     return os.path.expanduser(path)
 549
 550   def Exists(self, path):
 551     # This function largely exists so it can be overridden for testing.
 552     return os.path.exists(path)
 553
 554   def ReadFile(self, path):
 555     # This function largely exists so it can be overriden for testing.
 556     with open(path) as fp:
 557       return fp.read()
 558
 559   def RemoveFile(self, path):
 560     # This function largely exists so it can be overriden for testing.
 561     os.remove(path)
 562
 563   def TempFile(self, mode='w'):
 564     # This function largely exists so it can be overriden for testing.
 565     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 566
 567   def WriteFile(self, path, contents):
 568     # This function largely exists so it can be overriden for testing.
 569     with open(path, 'w') as fp:
 570       return fp.write(contents)
 571
 572
 573 class MBErr(Exception):
 574   pass
 575
 576
 577 if __name__ == '__main__':
 578   try:
 579     sys.exit(main(sys.argv[1:]))
 580   except MBErr as e:
 581     print(e)
 582     sys.exit(1)
 583   except KeyboardInterrupt:
 584     print("interrupted, exiting", stream=sys.stderr)
 585     sys.exit(130)