build/android/buildbot/bb_device_steps.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import collections
   7 import glob
   8 import hashlib
   9 import json
  10 import os
  11 import random
  12 import re
  13 import shutil
  14 import sys
  15
  16 import bb_utils
  17 import bb_annotations
  18
  19 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
  20 import provision_devices
  21 from devil.android import device_utils
  22 from pylib import constants
  23 from pylib.gtest import gtest_config
  24
  25 CHROME_SRC_DIR = bb_utils.CHROME_SRC
  26 DIR_BUILD_ROOT = os.path.dirname(CHROME_SRC_DIR)
  27 CHROME_OUT_DIR = bb_utils.CHROME_OUT_DIR
  28 BLINK_SCRIPTS_DIR = 'third_party/WebKit/Tools/Scripts'
  29
  30 SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave')
  31 LOGCAT_DIR = os.path.join(bb_utils.CHROME_OUT_DIR, 'logcat')
  32 GS_URL = 'https://storage.googleapis.com'
  33 GS_AUTH_URL = 'https://storage.cloud.google.com'
  34
  35 # Describes an instrumation test suite:
  36 #   test: Name of test we're running.
  37 #   apk: apk to be installed.
  38 #   apk_package: package for the apk to be installed.
  39 #   test_apk: apk to run tests on.
  40 #   test_data: data folder in format destination:source.
  41 #   host_driven_root: The host-driven test root directory.
  42 #   annotation: Annotation of the tests to include.
  43 #   exclude_annotation: The annotation of the tests to exclude.
  44 I_TEST = collections.namedtuple('InstrumentationTest', [
  45     'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'isolate_file_path',
  46     'host_driven_root', 'annotation', 'exclude_annotation', 'extra_flags'])
  47
  48
  49 def SrcPath(*path):
  50   return os.path.join(CHROME_SRC_DIR, *path)
  51
  52
  53 def I(name, apk, apk_package, test_apk, test_data, isolate_file_path=None,
  54       host_driven_root=None, annotation=None, exclude_annotation=None,
  55       extra_flags=None):
  56   return I_TEST(name, apk, apk_package, test_apk, test_data, isolate_file_path,
  57                 host_driven_root, annotation, exclude_annotation, extra_flags)
  58
  59 INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
  60     I('ContentShell',
  61       'ContentShell.apk',
  62       'org.chromium.content_shell_apk',
  63       'ContentShellTest',
  64       'content:content/test/data/android/device_files',
  65       isolate_file_path='content/content_shell_test_apk.isolate'),
  66     I('ChromePublic',
  67       'ChromePublic.apk',
  68       'org.chromium.chrome',
  69       'ChromePublicTest',
  70       'chrome:chrome/test/data/android/device_files',
  71       isolate_file_path='chrome/chrome_public_test_apk.isolate'),
  72     I('AndroidWebView',
  73       'AndroidWebView.apk',
  74       'org.chromium.android_webview.shell',
  75       'AndroidWebViewTest',
  76       'webview:android_webview/test/data/device_files',
  77       isolate_file_path='android_webview/android_webview_test_apk.isolate'),
  78     I('ChromeSyncShell',
  79       'ChromeSyncShell.apk',
  80       'org.chromium.chrome.browser.sync',
  81       'ChromeSyncShellTest',
  82       None),
  83     ])
  84
  85 InstallablePackage = collections.namedtuple('InstallablePackage', [
  86     'name', 'apk', 'apk_package'])
  87
  88 INSTALLABLE_PACKAGES = dict((package.name, package) for package in (
  89     [InstallablePackage(i.name, i.apk, i.apk_package)
  90      for i in INSTRUMENTATION_TESTS.itervalues()] +
  91     [InstallablePackage('ChromeDriverWebViewShell',
  92                         'ChromeDriverWebViewShell.apk',
  93                         'org.chromium.chromedriver_webview_shell')]))
  94
  95 VALID_TESTS = set([
  96     'base_junit_tests',
  97     'chromedriver',
  98     'components_browsertests',
  99     'gfx_unittests',
 100     'gl_unittests',
 101     'gpu',
 102     'python_unittests',
 103     'ui',
 104     'unit',
 105     'webkit',
 106     'webkit_layout'
 107 ])
 108
 109 RunCmd = bb_utils.RunCmd
 110
 111
 112 def _GetRevision(options):
 113   """Get the SVN revision number.
 114
 115   Args:
 116     options: options object.
 117
 118   Returns:
 119     The revision number.
 120   """
 121   revision = options.build_properties.get('got_revision')
 122   if not revision:
 123     revision = options.build_properties.get('revision', 'testing')
 124   return revision
 125
 126
 127 def _RunTest(options, cmd, suite):
 128   """Run test command with runtest.py.
 129
 130   Args:
 131     options: options object.
 132     cmd: the command to run.
 133     suite: test name.
 134   """
 135   property_args = bb_utils.EncodeProperties(options)
 136   args = [os.path.join(SLAVE_SCRIPTS_DIR, 'runtest.py')] + property_args
 137   args += ['--test-platform', 'android']
 138   if options.factory_properties.get('generate_gtest_json'):
 139     args.append('--generate-json-file')
 140     args += ['-o', 'gtest-results/%s' % suite,
 141              '--annotate', 'gtest',
 142              '--build-number', str(options.build_properties.get('buildnumber',
 143                                                                 '')),
 144              '--builder-name', options.build_properties.get('buildername', '')]
 145   if options.target == 'Release':
 146     args += ['--target', 'Release']
 147   else:
 148     args += ['--target', 'Debug']
 149   if options.flakiness_server:
 150     args += ['--flakiness-dashboard-server=%s' %
 151                 options.flakiness_server]
 152   args += cmd
 153   RunCmd(args, cwd=DIR_BUILD_ROOT)
 154
 155
 156 def RunTestSuites(options, suites, suites_options=None):
 157   """Manages an invocation of test_runner.py for gtests.
 158
 159   Args:
 160     options: options object.
 161     suites: List of suite names to run.
 162     suites_options: Command line options dictionary for particular suites.
 163                     For example,
 164                     {'content_browsertests', ['--num_retries=1', '--release']}
 165                     will add the options only to content_browsertests.
 166   """
 167
 168   if not suites_options:
 169     suites_options = {}
 170
 171   args = ['--verbose', '--blacklist-file', 'out/bad_devices.json']
 172   if options.target == 'Release':
 173     args.append('--release')
 174   if options.asan:
 175     args.append('--tool=asan')
 176   if options.gtest_filter:
 177     args.append('--gtest-filter=%s' % options.gtest_filter)
 178
 179   for suite in suites:
 180     bb_annotations.PrintNamedStep(suite)
 181     cmd = [suite] + args
 182     cmd += suites_options.get(suite, [])
 183     if suite == 'content_browsertests' or suite == 'components_browsertests':
 184       cmd.append('--num_retries=1')
 185     _RunTest(options, cmd, suite)
 186
 187
 188 def RunJunitSuite(suite):
 189   bb_annotations.PrintNamedStep(suite)
 190   RunCmd(['build/android/test_runner.py', 'junit', '-s', suite])
 191
 192
 193 def RunChromeDriverTests(options):
 194   """Run all the steps for running chromedriver tests."""
 195   bb_annotations.PrintNamedStep('chromedriver_annotation')
 196   RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py',
 197           '--android-packages=%s,%s,%s,%s' %
 198           ('chromium',
 199            'chrome_stable',
 200            'chrome_beta',
 201            'chromedriver_webview_shell'),
 202           '--revision=%s' % _GetRevision(options),
 203           '--update-log'])
 204
 205
 206 def InstallApk(options, test, print_step=False):
 207   """Install an apk to all phones.
 208
 209   Args:
 210     options: options object
 211     test: An I_TEST namedtuple
 212     print_step: Print a buildbot step
 213   """
 214   if print_step:
 215     bb_annotations.PrintNamedStep('install_%s' % test.name.lower())
 216
 217   args = [
 218       '--apk_package', test.apk_package,
 219       '--blacklist-file', 'out/bad_devices.json',
 220   ]
 221   if options.target == 'Release':
 222     args.append('--release')
 223   args.append(test.apk)
 224
 225   RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)
 226
 227
 228 def RunInstrumentationSuite(options, test, flunk_on_failure=True,
 229                             python_only=False, official_build=False):
 230   """Manages an invocation of test_runner.py for instrumentation tests.
 231
 232   Args:
 233     options: options object
 234     test: An I_TEST namedtuple
 235     flunk_on_failure: Flunk the step if tests fail.
 236     Python: Run only host driven Python tests.
 237     official_build: Run official-build tests.
 238   """
 239   bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower())
 240
 241   if test.apk:
 242     InstallApk(options, test)
 243   args = [
 244       '--test-apk', test.test_apk, '--verbose',
 245       '--blacklist-file', 'out/bad_devices.json'
 246   ]
 247   if test.test_data:
 248     args.extend(['--test_data', test.test_data])
 249   if options.target == 'Release':
 250     args.append('--release')
 251   if options.asan:
 252     args.append('--tool=asan')
 253   if options.flakiness_server:
 254     args.append('--flakiness-dashboard-server=%s' %
 255                 options.flakiness_server)
 256   if options.coverage_bucket:
 257     args.append('--coverage-dir=%s' % options.coverage_dir)
 258   if test.isolate_file_path:
 259     args.append('--isolate-file-path=%s' % test.isolate_file_path)
 260   if test.host_driven_root:
 261     args.append('--host-driven-root=%s' % test.host_driven_root)
 262   if test.annotation:
 263     args.extend(['-A', test.annotation])
 264   if test.exclude_annotation:
 265     args.extend(['-E', test.exclude_annotation])
 266   if test.extra_flags:
 267     args.extend(test.extra_flags)
 268   if python_only:
 269     args.append('-p')
 270   if official_build:
 271     # The option needs to be assigned 'True' as it does not have an action
 272     # associated with it.
 273     args.append('--official-build')
 274
 275   RunCmd(['build/android/test_runner.py', 'instrumentation'] + args,
 276          flunk_on_failure=flunk_on_failure)
 277
 278
 279 def RunWebkitLint():
 280   """Lint WebKit's TestExpectation files."""
 281   bb_annotations.PrintNamedStep('webkit_lint')
 282   RunCmd([SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'lint-test-expectations'))])
 283
 284
 285 def RunWebkitLayoutTests(options):
 286   """Run layout tests on an actual device."""
 287   bb_annotations.PrintNamedStep('webkit_tests')
 288   cmd_args = [
 289       '--no-show-results',
 290       '--no-new-test-results',
 291       '--full-results-html',
 292       '--clobber-old-results',
 293       '--exit-after-n-failures', '5000',
 294       '--exit-after-n-crashes-or-timeouts', '100',
 295       '--debug-rwt-logging',
 296       '--results-directory', '../layout-test-results',
 297       '--target', options.target,
 298       '--builder-name', options.build_properties.get('buildername', ''),
 299       '--build-number', str(options.build_properties.get('buildnumber', '')),
 300       '--master-name', 'ChromiumWebkit',  # TODO: Get this from the cfg.
 301       '--build-name', options.build_properties.get('buildername', ''),
 302       '--platform=android']
 303
 304   for flag in 'test_results_server', 'driver_name', 'additional_driver_flag':
 305     if flag in options.factory_properties:
 306       cmd_args.extend(['--%s' % flag.replace('_', '-'),
 307                        options.factory_properties.get(flag)])
 308
 309   for f in options.factory_properties.get('additional_expectations', []):
 310     cmd_args.extend(
 311         ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)])
 312
 313   # TODO(dpranke): Remove this block after
 314   # https://codereview.chromium.org/12927002/ lands.
 315   for f in options.factory_properties.get('additional_expectations_files', []):
 316     cmd_args.extend(
 317         ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)])
 318
 319   exit_code = RunCmd(
 320       [SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'run-webkit-tests'))] + cmd_args)
 321   if exit_code == 255: # test_run_results.UNEXPECTED_ERROR_EXIT_STATUS
 322     bb_annotations.PrintMsg('?? (crashed or hung)')
 323   elif exit_code == 254: # test_run_results.NO_DEVICES_EXIT_STATUS
 324     bb_annotations.PrintMsg('?? (no devices found)')
 325   elif exit_code == 253: # test_run_results.NO_TESTS_EXIT_STATUS
 326     bb_annotations.PrintMsg('?? (no tests found)')
 327   else:
 328     full_results_path = os.path.join('..', 'layout-test-results',
 329                                      'full_results.json')
 330     if os.path.exists(full_results_path):
 331       full_results = json.load(open(full_results_path))
 332       unexpected_passes, unexpected_failures, unexpected_flakes = (
 333           _ParseLayoutTestResults(full_results))
 334       if unexpected_failures:
 335         _PrintDashboardLink('failed', unexpected_failures.keys(),
 336                             max_tests=25)
 337       elif unexpected_passes:
 338         _PrintDashboardLink('unexpected passes', unexpected_passes.keys(),
 339                             max_tests=10)
 340       if unexpected_flakes:
 341         _PrintDashboardLink('unexpected flakes', unexpected_flakes.keys(),
 342                             max_tests=10)
 343
 344       if exit_code == 0 and (unexpected_passes or unexpected_flakes):
 345         # If exit_code != 0, RunCmd() will have already printed an error.
 346         bb_annotations.PrintWarning()
 347     else:
 348       bb_annotations.PrintError()
 349       bb_annotations.PrintMsg('?? (results missing)')
 350
 351   if options.factory_properties.get('archive_webkit_results', False):
 352     bb_annotations.PrintNamedStep('archive_webkit_results')
 353     base = 'https://storage.googleapis.com/chromium-layout-test-archives'
 354     builder_name = options.build_properties.get('buildername', '')
 355     build_number = str(options.build_properties.get('buildnumber', ''))
 356     results_link = '%s/%s/%s/layout-test-results/results.html' % (
 357         base, EscapeBuilderName(builder_name), build_number)
 358     bb_annotations.PrintLink('results', results_link)
 359     bb_annotations.PrintLink('(zip)', '%s/%s/%s/layout-test-results.zip' % (
 360         base, EscapeBuilderName(builder_name), build_number))
 361     gs_bucket = 'gs://chromium-layout-test-archives'
 362     RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'chromium',
 363                          'archive_layout_test_results.py'),
 364             '--results-dir', '../../layout-test-results',
 365             '--build-number', build_number,
 366             '--builder-name', builder_name,
 367             '--gs-bucket', gs_bucket],
 368             cwd=DIR_BUILD_ROOT)
 369
 370
 371 def _ParseLayoutTestResults(results):
 372   """Extract the failures from the test run."""
 373   # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results
 374   tests = _ConvertTrieToFlatPaths(results['tests'])
 375   failures = {}
 376   flakes = {}
 377   passes = {}
 378   for (test, result) in tests.iteritems():
 379     if result.get('is_unexpected'):
 380       actual_results = result['actual'].split()
 381       expected_results = result['expected'].split()
 382       if len(actual_results) > 1:
 383         # We report the first failure type back, even if the second
 384         # was more severe.
 385         if actual_results[1] in expected_results:
 386           flakes[test] = actual_results[0]
 387         else:
 388           failures[test] = actual_results[0]
 389       elif actual_results[0] == 'PASS':
 390         passes[test] = result
 391       else:
 392         failures[test] = actual_results[0]
 393
 394   return (passes, failures, flakes)
 395
 396
 397 def _ConvertTrieToFlatPaths(trie, prefix=None):
 398   """Flatten the trie of failures into a list."""
 399   # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results
 400   result = {}
 401   for name, data in trie.iteritems():
 402     if prefix:
 403       name = prefix + '/' + name
 404
 405     if len(data) and 'actual' not in data and 'expected' not in data:
 406       result.update(_ConvertTrieToFlatPaths(data, name))
 407     else:
 408       result[name] = data
 409
 410   return result
 411
 412
 413 def _PrintDashboardLink(link_text, tests, max_tests):
 414   """Add a link to the flakiness dashboard in the step annotations."""
 415   if len(tests) > max_tests:
 416     test_list_text = ' '.join(tests[:max_tests]) + ' and more'
 417   else:
 418     test_list_text = ' '.join(tests)
 419
 420   dashboard_base = ('http://test-results.appspot.com'
 421                     '/dashboards/flakiness_dashboard.html#'
 422                     'master=ChromiumWebkit&tests=')
 423
 424   bb_annotations.PrintLink('%d %s: %s' %
 425                            (len(tests), link_text, test_list_text),
 426                            dashboard_base + ','.join(tests))
 427
 428
 429 def EscapeBuilderName(builder_name):
 430   return re.sub('[ ()]', '_', builder_name)
 431
 432
 433 def SpawnLogcatMonitor():
 434   shutil.rmtree(LOGCAT_DIR, ignore_errors=True)
 435   bb_utils.SpawnCmd([
 436       os.path.join(CHROME_SRC_DIR, 'build', 'android', 'adb_logcat_monitor.py'),
 437       LOGCAT_DIR])
 438
 439   # Wait for logcat_monitor to pull existing logcat
 440   RunCmd(['sleep', '5'])
 441
 442
 443 def ProvisionDevices(options):
 444   bb_annotations.PrintNamedStep('provision_devices')
 445
 446   if not bb_utils.TESTING:
 447     # Restart adb to work around bugs, sleep to wait for usb discovery.
 448     device_utils.RestartServer()
 449     RunCmd(['sleep', '1'])
 450   provision_cmd = [
 451       'build/android/provision_devices.py', '-t', options.target,
 452       '--blacklist-file', 'out/bad_devices.json'
 453   ]
 454   if options.auto_reconnect:
 455     provision_cmd.append('--auto-reconnect')
 456   if options.skip_wipe:
 457     provision_cmd.append('--skip-wipe')
 458   if options.disable_location:
 459     provision_cmd.append('--disable-location')
 460   RunCmd(provision_cmd, halt_on_failure=True)
 461
 462
 463 def DeviceStatusCheck(options):
 464   bb_annotations.PrintNamedStep('device_status_check')
 465   cmd = [
 466       'build/android/buildbot/bb_device_status_check.py',
 467       '--blacklist-file', 'out/bad_devices.json',
 468   ]
 469   if options.restart_usb:
 470     cmd.append('--restart-usb')
 471   RunCmd(cmd, halt_on_failure=True)
 472
 473
 474 def GetDeviceSetupStepCmds():
 475   return [
 476       ('device_status_check', DeviceStatusCheck),
 477       ('provision_devices', ProvisionDevices),
 478   ]
 479
 480
 481 def RunUnitTests(options):
 482   suites = gtest_config.STABLE_TEST_SUITES
 483   if options.asan:
 484     suites = [s for s in suites
 485               if s not in gtest_config.ASAN_EXCLUDED_TEST_SUITES]
 486   RunTestSuites(options, suites)
 487
 488
 489 def RunInstrumentationTests(options):
 490   for test in INSTRUMENTATION_TESTS.itervalues():
 491     RunInstrumentationSuite(options, test)
 492
 493
 494 def RunWebkitTests(options):
 495   RunTestSuites(options, ['webkit_unit_tests', 'blink_heap_unittests'])
 496   RunWebkitLint()
 497
 498
 499 def RunGPUTests(options):
 500   revision = _GetRevision(options)
 501   builder_name = options.build_properties.get('buildername', 'noname')
 502
 503   bb_annotations.PrintNamedStep('pixel_tests')
 504   RunCmd(['content/test/gpu/run_gpu_test.py',
 505           'pixel', '-v',
 506           '--browser',
 507           'android-content-shell',
 508           '--build-revision',
 509           str(revision),
 510           '--upload-refimg-to-cloud-storage',
 511           '--refimg-cloud-storage-bucket',
 512           'chromium-gpu-archive/reference-images',
 513           '--os-type',
 514           'android',
 515           '--test-machine-name',
 516           EscapeBuilderName(builder_name),
 517           '--android-blacklist-file',
 518           'out/bad_devices.json'])
 519
 520   bb_annotations.PrintNamedStep('webgl_conformance_tests')
 521   RunCmd(['content/test/gpu/run_gpu_test.py', '-v',
 522           '--browser=android-content-shell', 'webgl_conformance',
 523           '--webgl-conformance-version=1.0.1',
 524           '--android-blacklist-file',
 525           'out/bad_devices.json'])
 526
 527   bb_annotations.PrintNamedStep('android_webview_webgl_conformance_tests')
 528   RunCmd(['content/test/gpu/run_gpu_test.py', '-v',
 529           '--browser=android-webview-shell', 'webgl_conformance',
 530           '--webgl-conformance-version=1.0.1',
 531           '--android-blacklist-file',
 532           'out/bad_devices.json'])
 533
 534   bb_annotations.PrintNamedStep('gpu_rasterization_tests')
 535   RunCmd(['content/test/gpu/run_gpu_test.py',
 536           'gpu_rasterization', '-v',
 537           '--browser',
 538           'android-content-shell',
 539           '--build-revision',
 540           str(revision),
 541           '--test-machine-name',
 542           EscapeBuilderName(builder_name),
 543           '--android-blacklist-file',
 544           'out/bad_devices.json'])
 545
 546
 547 def RunPythonUnitTests(_options):
 548   for suite in constants.PYTHON_UNIT_TEST_SUITES:
 549     bb_annotations.PrintNamedStep(suite)
 550     RunCmd(['build/android/test_runner.py', 'python', '-s', suite])
 551
 552
 553 def GetTestStepCmds():
 554   return [
 555       ('base_junit_tests',
 556           lambda _options: RunJunitSuite('base_junit_tests')),
 557       ('chromedriver', RunChromeDriverTests),
 558       ('components_browsertests',
 559           lambda options: RunTestSuites(options, ['components_browsertests'])),
 560       ('gfx_unittests',
 561           lambda options: RunTestSuites(options, ['gfx_unittests'])),
 562       ('gl_unittests',
 563           lambda options: RunTestSuites(options, ['gl_unittests'])),
 564       ('gpu', RunGPUTests),
 565       ('python_unittests', RunPythonUnitTests),
 566       ('ui', RunInstrumentationTests),
 567       ('unit', RunUnitTests),
 568       ('webkit', RunWebkitTests),
 569       ('webkit_layout', RunWebkitLayoutTests),
 570   ]
 571
 572
 573 def MakeGSPath(options, gs_base_dir):
 574   revision = _GetRevision(options)
 575   bot_id = options.build_properties.get('buildername', 'testing')
 576   randhash = hashlib.sha1(str(random.random())).hexdigest()
 577   gs_path = '%s/%s/%s/%s' % (gs_base_dir, bot_id, revision, randhash)
 578   # remove double slashes, happens with blank revisions and confuses gsutil
 579   gs_path = re.sub('/+', '/', gs_path)
 580   return gs_path
 581
 582 def UploadHTML(options, gs_base_dir, dir_to_upload, link_text,
 583                link_rel_path='index.html', gs_url=GS_URL):
 584   """Uploads directory at |dir_to_upload| to Google Storage and output a link.
 585
 586   Args:
 587     options: Command line options.
 588     gs_base_dir: The Google Storage base directory (e.g.
 589       'chromium-code-coverage/java')
 590     dir_to_upload: Absolute path to the directory to be uploaded.
 591     link_text: Link text to be displayed on the step.
 592     link_rel_path: Link path relative to |dir_to_upload|.
 593     gs_url: Google storage URL.
 594   """
 595   gs_path = MakeGSPath(options, gs_base_dir)
 596   RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-R', dir_to_upload, 'gs://%s' % gs_path])
 597   bb_annotations.PrintLink(link_text,
 598                            '%s/%s/%s' % (gs_url, gs_path, link_rel_path))
 599
 600
 601 def GenerateJavaCoverageReport(options):
 602   """Generates an HTML coverage report using EMMA and uploads it."""
 603   bb_annotations.PrintNamedStep('java_coverage_report')
 604
 605   coverage_html = os.path.join(options.coverage_dir, 'coverage_html')
 606   RunCmd(['build/android/generate_emma_html.py',
 607           '--coverage-dir', options.coverage_dir,
 608           '--metadata-dir', os.path.join(CHROME_OUT_DIR, options.target),
 609           '--cleanup',
 610           '--output', os.path.join(coverage_html, 'index.html')])
 611   return coverage_html
 612
 613
 614 def LogcatDump(options):
 615   # Print logcat, kill logcat monitor
 616   bb_annotations.PrintNamedStep('logcat_dump')
 617   logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt')
 618   RunCmd([SrcPath('build', 'android', 'adb_logcat_printer.py'),
 619           '--output-path', logcat_file, LOGCAT_DIR])
 620   gs_path = MakeGSPath(options, 'chromium-android/logcat_dumps')
 621   RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-z', 'txt', logcat_file,
 622           'gs://%s' % gs_path])
 623   bb_annotations.PrintLink('logcat dump', '%s/%s' % (GS_AUTH_URL, gs_path))
 624
 625
 626 def RunStackToolSteps(options):
 627   """Run stack tool steps.
 628
 629   Stack tool is run for logcat dump, optionally for ASAN.
 630   """
 631   bb_annotations.PrintNamedStep('Run stack tool with logcat dump')
 632   logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt')
 633   RunCmd([os.path.join(CHROME_SRC_DIR, 'third_party', 'android_platform',
 634           'development', 'scripts', 'stack'),
 635           '--more-info', logcat_file])
 636   if options.asan_symbolize:
 637     bb_annotations.PrintNamedStep('Run stack tool for ASAN')
 638     RunCmd([
 639         os.path.join(CHROME_SRC_DIR, 'build', 'android', 'asan_symbolize.py'),
 640         '-l', logcat_file])
 641
 642
 643 def GenerateTestReport(options):
 644   bb_annotations.PrintNamedStep('test_report')
 645   for report in glob.glob(
 646       os.path.join(CHROME_OUT_DIR, options.target, 'test_logs', '*.log')):
 647     RunCmd(['cat', report])
 648     os.remove(report)
 649
 650
 651 def MainTestWrapper(options):
 652   try:
 653     # Spawn logcat monitor
 654     SpawnLogcatMonitor()
 655
 656     # Run all device setup steps
 657     for _, cmd in GetDeviceSetupStepCmds():
 658       cmd(options)
 659
 660     if options.install:
 661       for i in options.install:
 662         install_obj = INSTALLABLE_PACKAGES[i]
 663         InstallApk(options, install_obj, print_step=True)
 664
 665     if options.test_filter:
 666       bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options)
 667
 668     if options.coverage_bucket:
 669       coverage_html = GenerateJavaCoverageReport(options)
 670       UploadHTML(options, '%s/java' % options.coverage_bucket, coverage_html,
 671                  'Coverage Report')
 672       shutil.rmtree(coverage_html, ignore_errors=True)
 673
 674     if options.experimental:
 675       RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)
 676
 677   finally:
 678     # Run all post test steps
 679     LogcatDump(options)
 680     if not options.disable_stack_tool:
 681       RunStackToolSteps(options)
 682     GenerateTestReport(options)
 683     # KillHostHeartbeat() has logic to check if heartbeat process is running,
 684     # and kills only if it finds the process is running on the host.
 685     provision_devices.KillHostHeartbeat()
 686     if options.cleanup:
 687       shutil.rmtree(os.path.join(CHROME_OUT_DIR, options.target),
 688           ignore_errors=True)
 689
 690
 691 def GetDeviceStepsOptParser():
 692   parser = bb_utils.GetParser()
 693   parser.add_option('--experimental', action='store_true',
 694                     help='Run experiemental tests')
 695   parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
 696                     action='append',
 697                     help=('Run a test suite. Test suites: "%s"' %
 698                           '", "'.join(VALID_TESTS)))
 699   parser.add_option('--gtest-filter',
 700                     help='Filter for running a subset of tests of a gtest test')
 701   parser.add_option('--asan', action='store_true', help='Run tests with asan.')
 702   parser.add_option('--install', metavar='<apk name>', action="append",
 703                     help='Install an apk by name')
 704   parser.add_option('--no-reboot', action='store_true',
 705                     help='Do not reboot devices during provisioning.')
 706   parser.add_option('--coverage-bucket',
 707                     help=('Bucket name to store coverage results. Coverage is '
 708                           'only run if this is set.'))
 709   parser.add_option('--restart-usb', action='store_true',
 710                     help='Restart usb ports before device status check.')
 711   parser.add_option(
 712       '--flakiness-server',
 713       help=('The flakiness dashboard server to which the results should be '
 714             'uploaded.'))
 715   parser.add_option(
 716       '--auto-reconnect', action='store_true',
 717       help='Push script to device which restarts adbd on disconnections.')
 718   parser.add_option('--skip-wipe', action='store_true',
 719                     help='Do not wipe devices during provisioning.')
 720   parser.add_option('--disable-location', action='store_true',
 721                     help='Disable location settings.')
 722   parser.add_option(
 723       '--logcat-dump-output',
 724       help='The logcat dump output will be "tee"-ed into this file')
 725   # During processing perf bisects, a seperate working directory created under
 726   # which builds are produced. Therefore we should look for relevent output
 727   # file under this directory.(/b/build/slave/<slave_name>/build/bisect/src/out)
 728   parser.add_option(
 729       '--chrome-output-dir',
 730       help='Chrome output directory to be used while bisecting.')
 731
 732   parser.add_option('--disable-stack-tool', action='store_true',
 733       help='Do not run stack tool.')
 734   parser.add_option('--asan-symbolize', action='store_true',
 735       help='Run stack tool for ASAN')
 736   parser.add_option('--cleanup', action='store_true',
 737       help='Delete out/<target> directory at the end of the run.')
 738   return parser
 739
 740
 741 def main(argv):
 742   parser = GetDeviceStepsOptParser()
 743   options, args = parser.parse_args(argv[1:])
 744
 745   if args:
 746     return sys.exit('Unused args %s' % args)
 747
 748   unknown_tests = set(options.test_filter) - VALID_TESTS
 749   if unknown_tests:
 750     return sys.exit('Unknown tests %s' % list(unknown_tests))
 751
 752   setattr(options, 'target', options.factory_properties.get('target', 'Debug'))
 753
 754   # pylint: disable=global-statement
 755   if options.chrome_output_dir:
 756     global CHROME_OUT_DIR
 757     global LOGCAT_DIR
 758     CHROME_OUT_DIR = options.chrome_output_dir
 759     LOGCAT_DIR = os.path.join(CHROME_OUT_DIR, 'logcat')
 760
 761   if options.coverage_bucket:
 762     setattr(options, 'coverage_dir',
 763             os.path.join(CHROME_OUT_DIR, options.target, 'coverage'))
 764
 765   MainTestWrapper(options)
 766
 767
 768 if __name__ == '__main__':
 769   sys.exit(main(sys.argv))