build/android/buildbot/bb_device_steps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import collections
   7 import glob
   8 import hashlib
   9 import multiprocessing
  10 import os
  11 import random
  12 import re
  13 import shutil
  14 import sys
  15
  16 import bb_utils
  17 import bb_annotations
  18
  19 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  20 import provision_devices
  21 from pylib import android_commands
  22 from pylib import constants
  23 from pylib.gtest import gtest_config
  24
  25 CHROME_SRC_DIR = bb_utils.CHROME_SRC
  26 CHROME_OUT_DIR = bb_utils.CHROME_OUT_DIR
  27 sys.path.append(os.path.join(
  28     CHROME_SRC_DIR, 'third_party', 'android_testrunner'))
  29 import errors
  30
  31
  32 SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave')
  33 LOGCAT_DIR = os.path.join(bb_utils.CHROME_OUT_DIR, 'logcat')
  34 GS_URL = 'https://storage.googleapis.com'
  35
  36 # Describes an instrumation test suite:
  37 #   test: Name of test we're running.
  38 #   apk: apk to be installed.
  39 #   apk_package: package for the apk to be installed.
  40 #   test_apk: apk to run tests on.
  41 #   test_data: data folder in format destination:source.
  42 #   host_driven_root: The host-driven test root directory.
  43 #   annotation: Annotation of the tests to include.
  44 #   exclude_annotation: The annotation of the tests to exclude.
  45 I_TEST = collections.namedtuple('InstrumentationTest', [
  46     'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root',
  47     'annotation', 'exclude_annotation', 'extra_flags'])
  48
  49 def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None,
  50       annotation=None, exclude_annotation=None, extra_flags=None):
  51   return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root,
  52                 annotation, exclude_annotation, extra_flags)
  53
  54 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
  55     I('ContentShell',
  56       'ContentShell.apk',
  57       'org.chromium.content_shell_apk',
  58       'ContentShellTest',
  59       'content:content/test/data/android/device_files'),
  60     I('ChromiumTestShell',
  61       'ChromiumTestShell.apk',
  62       'org.chromium.chrome.testshell',
  63       'ChromiumTestShellTest',
  64       'chrome:chrome/test/data/android/device_files',
  65       constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR),
  66     I('AndroidWebView',
  67       'AndroidWebView.apk',
  68       'org.chromium.android_webview.shell',
  69       'AndroidWebViewTest',
  70       'webview:android_webview/test/data/device_files'),
  71     ])
  72
  73 VALID_TESTS = set(['chromedriver', 'gpu', 'ui', 'unit', 'webkit',
  74                    'webkit_layout', 'webrtc'])
  75
  76 RunCmd = bb_utils.RunCmd
  77
  78
  79 # multiprocessing map_async requires a top-level function for pickle library.
  80 def RebootDeviceSafe(device):
  81   """Reboot a device, wait for it to start, and squelch timeout exceptions."""
  82   try:
  83     android_commands.AndroidCommands(device).Reboot(True)
  84   except errors.DeviceUnresponsiveError as e:
  85     return e
  86
  87
  88 def RebootDevices():
  89   """Reboot all attached and online devices."""
  90   # Early return here to avoid presubmit dependence on adb,
  91   # which might not exist in this checkout.
  92   if bb_utils.TESTING:
  93     return
  94   devices = android_commands.GetAttachedDevices(emulator=False)
  95   print 'Rebooting: %s' % devices
  96   if devices:
  97     pool = multiprocessing.Pool(len(devices))
  98     results = pool.map_async(RebootDeviceSafe, devices).get(99999)
  99
 100     for device, result in zip(devices, results):
 101       if result:
 102         print '%s failed to startup.' % device
 103
 104     if any(results):
 105       bb_annotations.PrintWarning()
 106     else:
 107       print 'Reboots complete.'
 108
 109
 110 def RunTestSuites(options, suites):
 111   """Manages an invocation of test_runner.py for gtests.
 112
 113   Args:
 114     options: options object.
 115     suites: List of suite names to run.
 116   """
 117   args = ['--verbose']
 118   if options.target == 'Release':
 119     args.append('--release')
 120   if options.asan:
 121     args.append('--tool=asan')
 122   for suite in suites:
 123     bb_annotations.PrintNamedStep(suite)
 124     cmd = ['build/android/test_runner.py', 'gtest', '-s', suite] + args
 125     if suite == 'content_browsertests':
 126       cmd.append('--num_retries=1')
 127     RunCmd(cmd)
 128
 129 def RunChromeDriverTests(_):
 130   """Run all the steps for running chromedriver tests."""
 131   bb_annotations.PrintNamedStep('chromedriver_annotation')
 132   RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py',
 133           '--android-packages=%s,%s,%s' %
 134            (constants.PACKAGE_INFO['chromium_test_shell'].package,
 135             constants.PACKAGE_INFO['chrome_stable'].package,
 136             constants.PACKAGE_INFO['chrome_beta'].package)])
 137
 138 def InstallApk(options, test, print_step=False):
 139   """Install an apk to all phones.
 140
 141   Args:
 142     options: options object
 143     test: An I_TEST namedtuple
 144     print_step: Print a buildbot step
 145   """
 146   if print_step:
 147     bb_annotations.PrintNamedStep('install_%s' % test.name.lower())
 148   # TODO(gkanwar): Quick hack to make sure AndroidWebViewTest.apk is replaced
 149   # before AndroidWebView.apk is. This can be removed once the bots cycle.
 150   args = ['--apk', '%s.apk' % test.test_apk]
 151   if options.target == 'Release':
 152     args.append('--release')
 153
 154   RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
 155
 156   args = ['--apk', test.apk, '--apk_package', test.apk_package]
 157   if options.target == 'Release':
 158     args.append('--release')
 159
 160   RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
 161
 162
 163 def RunInstrumentationSuite(options, test, flunk_on_failure=True,
 164                             python_only=False, official_build=False):
 165   """Manages an invocation of test_runner.py for instrumentation tests.
 166
 167   Args:
 168     options: options object
 169     test: An I_TEST namedtuple
 170     flunk_on_failure: Flunk the step if tests fail.
 171     Python: Run only host driven Python tests.
 172     official_build: Run official-build tests.
 173   """
 174   bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower())
 175
 176   InstallApk(options, test)
 177   args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
 178           '--verbose']
 179   if options.target == 'Release':
 180     args.append('--release')
 181   if options.asan:
 182     args.append('--tool=asan')
 183   if options.flakiness_server:
 184     args.append('--flakiness-dashboard-server=%s' %
 185                 options.flakiness_server)
 186   if options.coverage_bucket:
 187     args.append('--coverage-dir=%s' % options.coverage_dir)
 188   if test.host_driven_root:
 189     args.append('--host-driven-root=%s' % test.host_driven_root)
 190   if test.annotation:
 191     args.extend(['-A', test.annotation])
 192   if test.exclude_annotation:
 193     args.extend(['-E', test.exclude_annotation])
 194   if test.extra_flags:
 195     args.extend(test.extra_flags)
 196   if python_only:
 197     args.append('-p')
 198   if official_build:
 199     # The option needs to be assigned 'True' as it does not have an action
 200     # associated with it.
 201     args.append('--official-build')
 202
 203   RunCmd(['build/android/test_runner.py', 'instrumentation'] + args,
 204          flunk_on_failure=flunk_on_failure)
 205
 206
 207 def RunWebkitLint(target):
 208   """Lint WebKit's TestExpectation files."""
 209   bb_annotations.PrintNamedStep('webkit_lint')
 210   RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py',
 211           '--lint-test-files',
 212           '--chromium',
 213           '--target', target])
 214
 215
 216 def RunWebkitLayoutTests(options):
 217   """Run layout tests on an actual device."""
 218   bb_annotations.PrintNamedStep('webkit_tests')
 219   cmd_args = [
 220         '--no-show-results',
 221         '--no-new-test-results',
 222         '--full-results-html',
 223         '--clobber-old-results',
 224         '--exit-after-n-failures', '5000',
 225         '--exit-after-n-crashes-or-timeouts', '100',
 226         '--debug-rwt-logging',
 227         '--results-directory', '../layout-test-results',
 228         '--target', options.target,
 229         '--builder-name', options.build_properties.get('buildername', ''),
 230         '--build-number', str(options.build_properties.get('buildnumber', '')),
 231         '--master-name', 'ChromiumWebkit', # TODO: Get this from the cfg.
 232         '--build-name', options.build_properties.get('buildername', ''),
 233         '--platform=android']
 234
 235   for flag in 'test_results_server', 'driver_name', 'additional_drt_flag':
 236     if flag in options.factory_properties:
 237       cmd_args.extend(['--%s' % flag.replace('_', '-'),
 238                        options.factory_properties.get(flag)])
 239
 240   for f in options.factory_properties.get('additional_expectations', []):
 241     cmd_args.extend(
 242         ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)])
 243
 244   # TODO(dpranke): Remove this block after
 245   # https://codereview.chromium.org/12927002/ lands.
 246   for f in options.factory_properties.get('additional_expectations_files', []):
 247     cmd_args.extend(
 248         ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)])
 249
 250   RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args)
 251
 252   if options.factory_properties.get('archive_webkit_results', False):
 253     bb_annotations.PrintNamedStep('archive_webkit_results')
 254     base = 'https://storage.googleapis.com/chromium-layout-test-archives'
 255     builder_name = options.build_properties.get('buildername', '')
 256     build_number = str(options.build_properties.get('buildnumber', ''))
 257     bb_annotations.PrintLink('results',
 258         '%s/%s/%s/layout-test-results/results.html' % (
 259         base, EscapeBuilderName(builder_name), build_number))
 260     bb_annotations.PrintLink('(zip)', '%s/%s/%s/layout-test-results.zip' % (
 261         base, EscapeBuilderName(builder_name), build_number))
 262     gs_bucket = 'gs://chromium-layout-test-archives'
 263     RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
 264                          'archive_layout_test_results.py'),
 265         '--results-dir', '../../layout-test-results',
 266         '--build-dir', CHROME_OUT_DIR,
 267         '--build-number', build_number,
 268         '--builder-name', builder_name,
 269         '--gs-bucket', gs_bucket])
 270
 271
 272 def EscapeBuilderName(builder_name):
 273   return re.sub('[ ()]', '_', builder_name)
 274
 275
 276 def SpawnLogcatMonitor():
 277   shutil.rmtree(LOGCAT_DIR, ignore_errors=True)
 278   bb_utils.SpawnCmd([
 279       os.path.join(CHROME_SRC_DIR, 'build', 'android', 'adb_logcat_monitor.py'),
 280       LOGCAT_DIR])
 281
 282   # Wait for logcat_monitor to pull existing logcat
 283   RunCmd(['sleep', '5'])
 284
 285 def ProvisionDevices(options):
 286   bb_annotations.PrintNamedStep('provision_devices')
 287
 288   if not bb_utils.TESTING:
 289     # Restart adb to work around bugs, sleep to wait for usb discovery.
 290     adb = android_commands.AndroidCommands()
 291     adb.RestartAdbServer()
 292     RunCmd(['sleep', '1'])
 293
 294   if options.reboot:
 295     RebootDevices()
 296   provision_cmd = ['build/android/provision_devices.py', '-t', options.target]
 297   if options.auto_reconnect:
 298     provision_cmd.append('--auto-reconnect')
 299   RunCmd(provision_cmd)
 300
 301
 302 def DeviceStatusCheck(_):
 303   bb_annotations.PrintNamedStep('device_status_check')
 304   RunCmd(['build/android/buildbot/bb_device_status_check.py'],
 305          halt_on_failure=True)
 306
 307
 308 def GetDeviceSetupStepCmds():
 309   return [
 310     ('provision_devices', ProvisionDevices),
 311     ('device_status_check', DeviceStatusCheck),
 312   ]
 313
 314
 315 def RunUnitTests(options):
 316   RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)
 317
 318
 319 def RunInstrumentationTests(options):
 320   for test in INSTRUMENTATION_TESTS.itervalues():
 321     RunInstrumentationSuite(options, test)
 322
 323
 324 def RunWebkitTests(options):
 325   RunTestSuites(options, ['webkit_unit_tests'])
 326   RunWebkitLint(options.target)
 327
 328
 329 def RunWebRTCTests(options):
 330   RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES)
 331
 332
 333 def RunGPUTests(options):
 334   InstallApk(options, INSTRUMENTATION_TESTS['ContentShell'], False)
 335   bb_annotations.PrintNamedStep('gpu_tests')
 336   RunCmd(['content/test/gpu/run_gpu_test',
 337           '--browser=android-content-shell', 'pixel'])
 338
 339
 340 def GetTestStepCmds():
 341   return [
 342       ('chromedriver', RunChromeDriverTests),
 343       ('gpu', RunGPUTests),
 344       ('unit', RunUnitTests),
 345       ('ui', RunInstrumentationTests),
 346       ('webkit', RunWebkitTests),
 347       ('webkit_layout', RunWebkitLayoutTests),
 348       ('webrtc', RunWebRTCTests),
 349   ]
 350
 351
 352 def UploadHTML(options, gs_base_dir, dir_to_upload, link_text,
 353                link_rel_path='index.html', gs_url=GS_URL):
 354   """Uploads directory at |dir_to_upload| to Google Storage and output a link.
 355
 356   Args:
 357     options: Command line options.
 358     gs_base_dir: The Google Storage base directory (e.g.
 359       'chromium-code-coverage/java')
 360     dir_to_upload: Absolute path to the directory to be uploaded.
 361     link_text: Link text to be displayed on the step.
 362     link_rel_path: Link path relative to |dir_to_upload|.
 363     gs_url: Google storage URL.
 364   """
 365   revision = options.build_properties.get('got_revision')
 366   if not revision:
 367     revision = options.build_properties.get('revision', 'testing')
 368   bot_id = options.build_properties.get('buildername', 'testing')
 369   randhash = hashlib.sha1(str(random.random())).hexdigest()
 370   gs_path = '%s/%s/%s/%s' % (gs_base_dir, bot_id, revision, randhash)
 371   RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-R', dir_to_upload, 'gs://%s' % gs_path])
 372   bb_annotations.PrintLink(link_text,
 373       '%s/%s/%s' % (gs_url, gs_path, link_rel_path))
 374
 375
 376 def GenerateJavaCoverageReport(options):
 377   """Generates an HTML coverage report using EMMA and uploads it."""
 378   bb_annotations.PrintNamedStep('java_coverage_report')
 379
 380   coverage_html = os.path.join(options.coverage_dir, 'coverage_html')
 381   RunCmd(['build/android/generate_emma_html.py',
 382           '--coverage-dir', options.coverage_dir,
 383           '--metadata-dir', os.path.join(CHROME_OUT_DIR, options.target),
 384           '--cleanup',
 385           '--output', os.path.join(coverage_html, 'index.html')])
 386   return coverage_html
 387
 388
 389 def LogcatDump(options):
 390   # Print logcat, kill logcat monitor
 391   bb_annotations.PrintNamedStep('logcat_dump')
 392   logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log')
 393   with open(logcat_file, 'w') as f:
 394     RunCmd([
 395         os.path.join(CHROME_SRC_DIR, 'build', 'android',
 396                      'adb_logcat_printer.py'),
 397         LOGCAT_DIR], stdout=f)
 398   RunCmd(['cat', logcat_file])
 399
 400
 401 def GenerateTestReport(options):
 402   bb_annotations.PrintNamedStep('test_report')
 403   for report in glob.glob(
 404       os.path.join(CHROME_OUT_DIR, options.target, 'test_logs', '*.log')):
 405     RunCmd(['cat', report])
 406     os.remove(report)
 407
 408
 409 def MainTestWrapper(options):
 410   try:
 411     # Spawn logcat monitor
 412     SpawnLogcatMonitor()
 413
 414     # Run all device setup steps
 415     for _, cmd in GetDeviceSetupStepCmds():
 416       cmd(options)
 417
 418     if options.install:
 419       test_obj = INSTRUMENTATION_TESTS[options.install]
 420       InstallApk(options, test_obj, print_step=True)
 421
 422     if options.test_filter:
 423       bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options)
 424
 425     if options.coverage_bucket:
 426       coverage_html = GenerateJavaCoverageReport(options)
 427       UploadHTML(options, '%s/java' % options.coverage_bucket, coverage_html,
 428                  'Coverage Report')
 429
 430     if options.experimental:
 431       RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)
 432
 433   finally:
 434     # Run all post test steps
 435     LogcatDump(options)
 436     GenerateTestReport(options)
 437     # KillHostHeartbeat() has logic to check if heartbeat process is running,
 438     # and kills only if it finds the process is running on the host.
 439     provision_devices.KillHostHeartbeat()
 440
 441
 442 def GetDeviceStepsOptParser():
 443   parser = bb_utils.GetParser()
 444   parser.add_option('--experimental', action='store_true',
 445                     help='Run experiemental tests')
 446   parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
 447                     action='append',
 448                     help=('Run a test suite. Test suites: "%s"' %
 449                           '", "'.join(VALID_TESTS)))
 450   parser.add_option('--asan', action='store_true', help='Run tests with asan.')
 451   parser.add_option('--install', metavar='<apk name>',
 452                     help='Install an apk by name')
 453   parser.add_option('--reboot', action='store_true',
 454                     help='Reboot devices before running tests')
 455   parser.add_option('--coverage-bucket',
 456                     help=('Bucket name to store coverage results. Coverage is '
 457                           'only run if this is set.'))
 458   parser.add_option(
 459       '--flakiness-server',
 460       help='The flakiness dashboard server to which the results should be '
 461            'uploaded.')
 462   parser.add_option(
 463       '--auto-reconnect', action='store_true',
 464       help='Push script to device which restarts adbd on disconnections.')
 465   parser.add_option(
 466       '--logcat-dump-output',
 467       help='The logcat dump output will be "tee"-ed into this file')
 468
 469   return parser
 470
 471
 472 def main(argv):
 473   parser = GetDeviceStepsOptParser()
 474   options, args = parser.parse_args(argv[1:])
 475
 476   if args:
 477     return sys.exit('Unused args %s' % args)
 478
 479   unknown_tests = set(options.test_filter) - VALID_TESTS
 480   if unknown_tests:
 481     return sys.exit('Unknown tests %s' % list(unknown_tests))
 482
 483   setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
 484   if options.coverage_bucket:
 485     setattr(options, 'coverage_dir',
 486             os.path.join(CHROME_OUT_DIR, options.target, 'coverage'))
 487
 488   MainTestWrapper(options)
 489
 490
 491 if __name__ == '__main__':
 492   sys.exit(main(sys.argv))