build/android/buildbot/bb_run_bot.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 import collections
   8 import copy
   9 import json
  10 import os
  11 import pipes
  12 import re
  13 import subprocess
  14 import sys
  15
  16 import bb_utils
  17
  18 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  19 from pylib import constants
  20
  21
  22 CHROMIUM_COVERAGE_BUCKET = 'chromium-code-coverage'
  23
  24 _BotConfig = collections.namedtuple(
  25     'BotConfig', ['bot_id', 'host_obj', 'test_obj'])
  26
  27 HostConfig = collections.namedtuple(
  28     'HostConfig',
  29     ['script', 'host_steps', 'extra_args', 'extra_gyp_defines', 'target_arch'])
  30
  31 TestConfig = collections.namedtuple('Tests', ['script', 'tests', 'extra_args'])
  32
  33
  34 def BotConfig(bot_id, host_object, test_object=None):
  35   return _BotConfig(bot_id, host_object, test_object)
  36
  37
  38 def DictDiff(d1, d2):
  39   diff = []
  40   for key in sorted(set(d1.keys() + d2.keys())):
  41     if key in d1 and d1[key] != d2.get(key):
  42       diff.append('- %s=%s' % (key, pipes.quote(d1[key])))
  43     if key in d2 and d2[key] != d1.get(key):
  44       diff.append('+ %s=%s' % (key, pipes.quote(d2[key])))
  45   return '\n'.join(diff)
  46
  47
  48 def GetEnvironment(host_obj, testing, extra_env_vars=None):
  49   init_env = dict(os.environ)
  50   init_env['GYP_GENERATORS'] = 'ninja'
  51   if extra_env_vars:
  52     init_env.update(extra_env_vars)
  53   envsetup_cmd = '. build/android/envsetup.sh'
  54   if testing:
  55     # Skip envsetup to avoid presubmit dependence on android deps.
  56     print 'Testing mode - skipping "%s"' % envsetup_cmd
  57     envsetup_cmd = ':'
  58   else:
  59     print 'Running %s' % envsetup_cmd
  60   proc = subprocess.Popen(['bash', '-exc',
  61     envsetup_cmd + ' >&2; python build/android/buildbot/env_to_json.py'],
  62     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  63     cwd=bb_utils.CHROME_SRC, env=init_env)
  64   json_env, envsetup_output = proc.communicate()
  65   if proc.returncode != 0:
  66     print >> sys.stderr, 'FATAL Failure in envsetup.'
  67     print >> sys.stderr, envsetup_output
  68     sys.exit(1)
  69   env = json.loads(json_env)
  70   env['GYP_DEFINES'] = env.get('GYP_DEFINES', '') + \
  71       ' OS=android fastbuild=1 use_goma=1 gomadir=%s' % bb_utils.GOMA_DIR
  72   if host_obj.target_arch:
  73     env['GYP_DEFINES'] += ' target_arch=%s' % host_obj.target_arch
  74   extra_gyp = host_obj.extra_gyp_defines
  75   if extra_gyp:
  76     env['GYP_DEFINES'] += ' %s' % extra_gyp
  77     if re.search('(asan|clang)=1', extra_gyp):
  78       env.pop('CXX_target', None)
  79
  80   # Bots checkout chrome in /b/build/slave/<name>/build/src
  81   build_internal_android = os.path.abspath(os.path.join(
  82       bb_utils.CHROME_SRC, '..', '..', '..', '..', '..', 'build_internal',
  83       'scripts', 'slave', 'android'))
  84   if os.path.exists(build_internal_android):
  85     env['PATH'] = os.pathsep.join([build_internal_android, env['PATH']])
  86   return env
  87
  88
  89 def GetCommands(options, bot_config):
  90   """Get a formatted list of commands.
  91
  92   Args:
  93     options: Options object.
  94     bot_config: A BotConfig named tuple.
  95     host_step_script: Host step script.
  96     device_step_script: Device step script.
  97   Returns:
  98     list of Command objects.
  99   """
 100   property_args = bb_utils.EncodeProperties(options)
 101   commands = [[bot_config.host_obj.script,
 102                '--steps=%s' % ','.join(bot_config.host_obj.host_steps)] +
 103               property_args + (bot_config.host_obj.extra_args or [])]
 104
 105   test_obj = bot_config.test_obj
 106   if test_obj:
 107     run_test_cmd = [test_obj.script] + property_args
 108     for test in test_obj.tests:
 109       run_test_cmd.extend(['-f', test])
 110     if test_obj.extra_args:
 111       run_test_cmd.extend(test_obj.extra_args)
 112     commands.append(run_test_cmd)
 113   return commands
 114
 115
 116 def GetBotStepMap():
 117   compile_step = ['compile']
 118   chrome_proxy_tests = ['chrome_proxy']
 119   chrome_sync_shell_tests = ['sync']
 120   std_host_tests = ['check_webview_licenses', 'findbugs']
 121   std_build_steps = ['compile', 'zip_build']
 122   std_test_steps = ['extract_build']
 123   std_tests = ['ui', 'unit', 'mojo']
 124   telemetry_tests = ['telemetry_perf_unittests']
 125   flakiness_server = (
 126       '--flakiness-server=%s' % constants.UPSTREAM_FLAKINESS_SERVER)
 127   experimental = ['--experimental']
 128   bisect_chrome_output_dir = os.path.abspath(
 129       os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir,
 130                    os.pardir, 'bisect', 'src', 'out'))
 131   B = BotConfig
 132   H = (lambda steps, extra_args=None, extra_gyp=None, target_arch=None :
 133        HostConfig('build/android/buildbot/bb_host_steps.py', steps, extra_args,
 134                   extra_gyp, target_arch))
 135   T = (lambda tests, extra_args=None :
 136        TestConfig('build/android/buildbot/bb_device_steps.py', tests,
 137                   extra_args))
 138
 139   bot_configs = [
 140       # Main builders
 141       B('main-builder-dbg', H(std_build_steps + std_host_tests)),
 142       B('main-builder-rel', H(std_build_steps)),
 143       B('main-clang-builder',
 144         H(compile_step, extra_gyp='clang=1 component=shared_library')),
 145       B('main-clobber', H(compile_step)),
 146       B('main-tests-rel', H(std_test_steps),
 147         T(std_tests + telemetry_tests + chrome_proxy_tests,
 148           ['--cleanup', flakiness_server])),
 149       B('main-tests', H(std_test_steps),
 150         T(std_tests,['--cleanup', flakiness_server])),
 151
 152       # Other waterfalls
 153       B('asan-builder-tests', H(compile_step,
 154                                 extra_gyp='asan=1 component=shared_library'),
 155         T(std_tests, ['--asan', '--asan-symbolize'])),
 156       B('blink-try-builder', H(compile_step)),
 157       B('chromedriver-fyi-tests-dbg', H(std_test_steps),
 158         T(['chromedriver'], ['--install=ChromeShell', '--skip-wipe',
 159           '--cleanup'])),
 160       B('fyi-x86-builder-dbg',
 161         H(compile_step + std_host_tests, experimental, target_arch='x86')),
 162       B('fyi-builder-dbg',
 163         H(std_build_steps + std_host_tests, experimental,
 164           extra_gyp='emma_coverage=1')),
 165       B('x86-builder-dbg',
 166         H(compile_step + std_host_tests, target_arch='x86')),
 167       B('fyi-builder-rel', H(std_build_steps,  experimental)),
 168       B('fyi-tests', H(std_test_steps),
 169         T(std_tests + chrome_sync_shell_tests,
 170                       ['--experimental', flakiness_server,
 171                       '--coverage-bucket', CHROMIUM_COVERAGE_BUCKET,
 172                       '--cleanup'])),
 173       B('fyi-component-builder-tests-dbg',
 174         H(compile_step, extra_gyp='component=shared_library'),
 175         T(std_tests, ['--experimental', flakiness_server])),
 176       B('gpu-builder-tests-dbg',
 177         H(compile_step),
 178         T(['gpu'], ['--install=ContentShell'])),
 179       # Pass empty T([]) so that logcat monitor and device status check are run.
 180       B('perf-bisect-builder-tests-dbg',
 181         H(['bisect_perf_regression']),
 182         T([], ['--chrome-output-dir', bisect_chrome_output_dir])),
 183       B('perf-tests-rel', H(std_test_steps),
 184         T([], ['--install=ChromeShell', '--cleanup'])),
 185       B('webkit-latest-webkit-tests', H(std_test_steps),
 186         T(['webkit_layout', 'webkit'], ['--cleanup', '--auto-reconnect'])),
 187       B('webkit-latest-contentshell', H(compile_step),
 188         T(['webkit_layout'], ['--auto-reconnect'])),
 189       B('builder-unit-tests', H(compile_step), T(['unit'])),
 190       B('webrtc-chromium-builder',
 191         H(std_build_steps,
 192           extra_args=['--build-targets=android_builder_chromium_webrtc'])),
 193       B('webrtc-native-builder',
 194         H(std_build_steps,
 195           extra_args=['--build-targets=android_builder_webrtc'],
 196           extra_gyp='include_tests=1 enable_tracing=1')),
 197       B('webrtc-chromium-tests', H(std_test_steps),
 198         T(['webrtc_chromium'],
 199           [flakiness_server, '--gtest-filter=WebRtc*', '--cleanup'])),
 200       B('webrtc-native-tests', H(std_test_steps),
 201         T(['webrtc_native'], ['--cleanup', flakiness_server])),
 202
 203       # Generic builder config (for substring match).
 204       B('builder', H(std_build_steps)),
 205   ]
 206
 207   bot_map = dict((config.bot_id, config) for config in bot_configs)
 208
 209   # These bots have identical configuration to ones defined earlier.
 210   copy_map = [
 211       ('lkgr-clobber', 'main-clobber'),
 212       ('try-builder-dbg', 'main-builder-dbg'),
 213       ('try-builder-rel', 'main-builder-rel'),
 214       ('try-clang-builder', 'main-clang-builder'),
 215       ('try-fyi-builder-dbg', 'fyi-builder-dbg'),
 216       ('try-x86-builder-dbg', 'x86-builder-dbg'),
 217       ('try-tests-rel', 'main-tests-rel'),
 218       ('try-tests', 'main-tests'),
 219       ('try-fyi-tests', 'fyi-tests'),
 220       ('webkit-latest-tests', 'main-tests'),
 221   ]
 222   for to_id, from_id in copy_map:
 223     assert to_id not in bot_map
 224     # pylint: disable=W0212
 225     bot_map[to_id] = copy.deepcopy(bot_map[from_id])._replace(bot_id=to_id)
 226
 227     # Trybots do not upload to flakiness dashboard. They should be otherwise
 228     # identical in configuration to their trunk building counterparts.
 229     test_obj = bot_map[to_id].test_obj
 230     if to_id.startswith('try') and test_obj:
 231       extra_args = test_obj.extra_args
 232       if extra_args and flakiness_server in extra_args:
 233         extra_args.remove(flakiness_server)
 234   return bot_map
 235
 236
 237 # Return an object from the map, looking first for an exact id match.
 238 # If this fails, look for an id which is a substring of the specified id.
 239 # Choose the longest of all substring matches.
 240 # pylint: disable=W0622
 241 def GetBestMatch(id_map, id):
 242   config = id_map.get(id)
 243   if not config:
 244     substring_matches = filter(lambda x: x in id, id_map.iterkeys())
 245     if substring_matches:
 246       max_id = max(substring_matches, key=len)
 247       print 'Using config from id="%s" (substring match).' % max_id
 248       config = id_map[max_id]
 249   return config
 250
 251
 252 def GetRunBotOptParser():
 253   parser = bb_utils.GetParser()
 254   parser.add_option('--bot-id', help='Specify bot id directly.')
 255   parser.add_option('--testing', action='store_true',
 256                     help='For testing: print, but do not run commands')
 257
 258   return parser
 259
 260
 261 def GetBotConfig(options, bot_step_map):
 262   bot_id = options.bot_id or options.factory_properties.get('android_bot_id')
 263   if not bot_id:
 264     print (sys.stderr,
 265            'A bot id must be specified through option or factory_props.')
 266     return
 267
 268   bot_config = GetBestMatch(bot_step_map, bot_id)
 269   if not bot_config:
 270     print 'Error: config for id="%s" cannot be inferred.' % bot_id
 271   return bot_config
 272
 273
 274 def RunBotCommands(options, commands, env):
 275   print 'Environment changes:'
 276   print DictDiff(dict(os.environ), env)
 277
 278   for command in commands:
 279     print bb_utils.CommandToString(command)
 280     sys.stdout.flush()
 281     if options.testing:
 282       env['BUILDBOT_TESTING'] = '1'
 283     return_code = subprocess.call(command, cwd=bb_utils.CHROME_SRC, env=env)
 284     if return_code != 0:
 285       return return_code
 286
 287
 288 def main(argv):
 289   proc = subprocess.Popen(
 290       ['/bin/hostname', '-f'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 291   hostname_stdout, hostname_stderr = proc.communicate()
 292   if proc.returncode == 0:
 293     print 'Running on: ' + hostname_stdout
 294   else:
 295     print >> sys.stderr, 'WARNING: failed to run hostname'
 296     print >> sys.stderr, hostname_stdout
 297     print >> sys.stderr, hostname_stderr
 298     sys.exit(1)
 299
 300   parser = GetRunBotOptParser()
 301   options, args = parser.parse_args(argv[1:])
 302   if args:
 303     parser.error('Unused args: %s' % args)
 304
 305   bot_config = GetBotConfig(options, GetBotStepMap())
 306   if not bot_config:
 307     sys.exit(1)
 308
 309   print 'Using config:', bot_config
 310
 311   commands = GetCommands(options, bot_config)
 312   for command in commands:
 313     print 'Will run: ', bb_utils.CommandToString(command)
 314   print
 315
 316   env = GetEnvironment(bot_config.host_obj, options.testing)
 317   return RunBotCommands(options, commands, env)
 318
 319
 320 if __name__ == '__main__':
 321   sys.exit(main(sys.argv))