components/cronet/android/test/javaperftests/run.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """This script runs an automated Cronet performance benchmark.
   7
   8 This script:
   9 1. Sets up "USB reverse tethering" which allow network traffic to flow from
  10    an Android device connected to the host machine via a USB cable.
  11 2. Starts HTTP and QUIC servers on the host machine.
  12 3. Installs an Android app on the attached Android device and runs it.
  13 4. Collects the results from the app.
  14
  15 Prerequisites:
  16 1. A rooted (i.e. "adb root" succeeds) Android device connected via a USB cable
  17    to the host machine (i.e. the computer running this script).
  18 2. quic_server and quic_client have been built for the host machine, e.g. via:
  19      ./build/gyp_chromium
  20      ninja -C out/Release quic_server quic_client
  21 3. cronet_perf_test_apk has been built for the Android device, e.g. via:
  22      ./components/cronet/tools/cr_cronet.py gyp
  23      ninja -C out/Release cronet_perf_test_apk
  24
  25 Invocation:
  26 ./run.py
  27
  28 Output:
  29 Benchmark timings are output by telemetry to stdout and written to
  30 ./results.html
  31
  32 """
  33
  34 import json
  35 import os
  36 import shutil
  37 import subprocess
  38 import sys
  39 import tempfile
  40 from time import sleep
  41 import urllib
  42
  43 REPOSITORY_ROOT = os.path.abspath(os.path.join(
  44     os.path.dirname(__file__), '..', '..', '..', '..', '..'))
  45
  46 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools/telemetry'))
  47 sys.path.append(os.path.join(REPOSITORY_ROOT, 'build/android'))
  48
  49 import lighttpd_server
  50 from pylib import constants
  51 from pylib import pexpect
  52 from pylib.device import device_utils
  53 from pylib.device import intent
  54 from telemetry import android
  55 from telemetry import benchmark
  56 from telemetry import benchmark_runner
  57 from telemetry import story
  58 from telemetry.internal import forwarders
  59 from telemetry.internal.forwarders import android_forwarder
  60 from telemetry.value import scalar
  61 from telemetry.web_perf import timeline_based_measurement
  62
  63 BUILD_TYPE = 'Release'
  64 BUILD_DIR = os.path.join(REPOSITORY_ROOT, 'out', BUILD_TYPE)
  65 QUIC_SERVER = os.path.join(BUILD_DIR, 'quic_server')
  66 QUIC_CLIENT = os.path.join(BUILD_DIR, 'quic_client')
  67 APP_APK = os.path.join(BUILD_DIR, 'apks', 'CronetPerfTest.apk')
  68 APP_PACKAGE = 'org.chromium.net'
  69 APP_ACTIVITY = '.CronetPerfTestActivity'
  70 APP_ACTION = 'android.intent.action.MAIN'
  71 BENCHMARK_CONFIG = {
  72   # Control various metric recording for further investigation.
  73   'CAPTURE_NETLOG': False,
  74   'CAPTURE_TRACE': False,
  75   'CAPTURE_SAMPLED_TRACE': False,
  76   # While running Cronet Async API benchmarks, indicate if callbacks should be
  77   # run on network thread rather than posted back to caller thread.  This allows
  78   # measuring if thread-hopping overhead is significant.
  79   'CRONET_ASYNC_USE_NETWORK_THREAD': False,
  80   # A small resource for device to fetch from host.
  81   'SMALL_RESOURCE': 'small.html',
  82   'SMALL_RESOURCE_SIZE': 26,
  83   # Number of times to fetch SMALL_RESOURCE.
  84   'SMALL_ITERATIONS': 1000,
  85   # A large resource for device to fetch from host.
  86   'LARGE_RESOURCE': 'large.html',
  87   'LARGE_RESOURCE_SIZE': 10000026,
  88   # Number of times to fetch LARGE_RESOURCE.
  89   'LARGE_ITERATIONS': 4,
  90   # An on-device file containing benchmark timings.  Written by benchmark app.
  91   'RESULTS_FILE': '/data/data/' + APP_PACKAGE + '/results.txt',
  92   # An on-device file whose presence indicates benchmark app has terminated.
  93   'DONE_FILE': '/data/data/' + APP_PACKAGE + '/done.txt',
  94   # Ports of HTTP and QUIC servers on host.
  95   'HTTP_PORT': 9000,
  96   'QUIC_PORT': 9001,
  97   # Maximum read/write buffer size to use.
  98   'MAX_BUFFER_SIZE': 16384,
  99 }
 100 # Add benchmark config to global state for easy access.
 101 globals().update(BENCHMARK_CONFIG)
 102
 103
 104 def GetDevice():
 105   devices = device_utils.DeviceUtils.HealthyDevices()
 106   assert len(devices) == 1
 107   return devices[0]
 108
 109
 110 def GetForwarderFactory(device):
 111   return android_forwarder.AndroidForwarderFactory(device, True)
 112
 113
 114 def GetServersHost(device):
 115   return GetForwarderFactory(device).host_ip
 116
 117
 118 def GetHttpServerURL(device, resource):
 119   return 'http://%s:%d/%s' % (GetServersHost(device), HTTP_PORT, resource)
 120
 121
 122 class CronetPerfTestAndroidStory(android.AndroidStory):
 123   # Android AppStory implementation wrapping CronetPerfTest app.
 124   # Launches Cronet perf test app and waits for execution to complete
 125   # by waiting for presence of DONE_FILE.
 126
 127   def __init__(self, device):
 128     self._device = device
 129     device.RunShellCommand('rm %s' % DONE_FILE)
 130     config = BENCHMARK_CONFIG
 131     config['HOST'] = GetServersHost(device)
 132     start_intent = intent.Intent(
 133         package=APP_PACKAGE,
 134         activity=APP_ACTIVITY,
 135         action=APP_ACTION,
 136         # |config| maps from configuration value names to the configured values.
 137         # |config| is encoded as URL parameter names and values and passed to
 138         # the Cronet perf test app via the Intent data field.
 139         data='http://dummy/?'+urllib.urlencode(config),
 140         extras=None,
 141         category=None)
 142     super(CronetPerfTestAndroidStory, self).__init__(
 143         start_intent, name='CronetPerfTest',
 144         # No reason to wait for app; Run() will wait for results.  By default
 145         # StartActivity will timeout waiting for CronetPerfTest, so override
 146         # |is_app_ready_predicate| to not wait.
 147         is_app_ready_predicate=lambda app: True)
 148
 149   def Run(self, shared_user_story_state):
 150     while not self._device.FileExists(DONE_FILE):
 151       sleep(1.0)
 152
 153
 154 class CronetPerfTestStorySet(story.StorySet):
 155
 156   def __init__(self, device):
 157     super(CronetPerfTestStorySet, self).__init__()
 158     # Create and add Cronet perf test AndroidStory.
 159     self.AddStory(CronetPerfTestAndroidStory(device))
 160
 161
 162 class CronetPerfTestMeasurement(
 163     timeline_based_measurement.TimelineBasedMeasurement):
 164   # For now AndroidStory's SharedAppState works only with
 165   # TimelineBasedMeasurements, so implement one that just forwards results from
 166   # Cronet perf test app.
 167
 168   def __init__(self, device, options):
 169     super(CronetPerfTestMeasurement, self).__init__(options)
 170     self._device = device
 171
 172   def WillRunStoryForPageTest(self, tracing_controller):
 173     # Skip parent implementation which doesn't apply to Cronet perf test app as
 174     # it is not a browser with a timeline interface.
 175     pass
 176
 177   def Measure(self, tracing_controller, results):
 178     # Reads results from |RESULTS_FILE| on target and adds to |results|.
 179     jsonResults = json.loads(self._device.ReadFile(RESULTS_FILE))
 180     for test in jsonResults:
 181       results.AddValue(scalar.ScalarValue(results.current_page, test,
 182           'ms', jsonResults[test]))
 183
 184
 185 @benchmark.Enabled('android')
 186 class CronetPerfTestBenchmark(benchmark.Benchmark):
 187   # Benchmark implementation spawning off Cronet perf test measurement and
 188   # StorySet.
 189
 190   def __init__(self, max_failures=None):
 191     super(CronetPerfTestBenchmark, self).__init__(max_failures)
 192     self._device = GetDevice()
 193
 194   def CreatePageTest(self, options):
 195     return CronetPerfTestMeasurement(self._device, options)
 196
 197   def CreateStorySet(self, options):
 198     return CronetPerfTestStorySet(self._device)
 199
 200
 201 class QuicServer:
 202
 203   def __init__(self, quic_server_doc_root):
 204     self._process = None
 205     self._quic_server_doc_root = quic_server_doc_root
 206
 207   def StartupQuicServer(self, device):
 208     self._process = pexpect.spawn(QUIC_SERVER,
 209                                   ['--quic_in_memory_cache_dir=%s' %
 210                                       self._quic_server_doc_root,
 211                                    '--port=%d' % QUIC_PORT])
 212     assert self._process != None
 213     # Wait for quic_server to start serving.
 214     waited_s = 0
 215     while subprocess.call([QUIC_CLIENT,
 216                            '--host=%s' % GetServersHost(device),
 217                            '--port=%d' % QUIC_PORT,
 218                            'http://%s:%d/%s' % (GetServersHost(device),
 219                                                 QUIC_PORT, SMALL_RESOURCE)],
 220                           stdout=open(os.devnull, 'w')) != 0:
 221       sleep(0.1)
 222       waited_s += 0.1
 223       assert waited_s < 5, "quic_server failed to start after %fs" % waited_s
 224
 225   def ShutdownQuicServer(self):
 226     if self._process:
 227       self._process.terminate()
 228
 229
 230 def GenerateHttpTestResources():
 231   http_server_doc_root = tempfile.mkdtemp()
 232   # Create a small test file to serve.
 233   small_file_name = os.path.join(http_server_doc_root, SMALL_RESOURCE)
 234   small_file = open(small_file_name, 'wb')
 235   small_file.write('<html><body></body></html>');
 236   small_file.close()
 237   assert SMALL_RESOURCE_SIZE == os.path.getsize(small_file_name)
 238   # Create a large (10MB) test file to serve.
 239   large_file_name = os.path.join(http_server_doc_root, LARGE_RESOURCE)
 240   large_file = open(large_file_name, 'wb')
 241   large_file.write('<html><body>');
 242   for i in range(0, 1000000):
 243     large_file.write('1234567890');
 244   large_file.write('</body></html>');
 245   large_file.close()
 246   assert LARGE_RESOURCE_SIZE == os.path.getsize(large_file_name)
 247   return http_server_doc_root
 248
 249
 250 def GenerateQuicTestResources(device):
 251   quic_server_doc_root = tempfile.mkdtemp()
 252   # Use wget to build up fake QUIC in-memory cache dir for serving.
 253   # quic_server expects the dir/file layout that wget produces.
 254   for resource in [SMALL_RESOURCE, LARGE_RESOURCE]:
 255     assert subprocess.Popen(['wget', '-p', '-q', '--save-headers',
 256                              GetHttpServerURL(device, resource)],
 257                             cwd=quic_server_doc_root).wait() == 0
 258   # wget places results in host:port directory.  Adjust for QUIC port.
 259   os.rename(os.path.join(quic_server_doc_root,
 260                          "%s:%d" % (GetServersHost(device), HTTP_PORT)),
 261             os.path.join(quic_server_doc_root,
 262                          "%s:%d" % (GetServersHost(device), QUIC_PORT)))
 263   return quic_server_doc_root
 264
 265
 266 def GenerateLighttpdConfig(config_file, http_server_doc_root, http_server):
 267   # Must create customized config file to allow overriding the server.bind
 268   # setting.
 269   config_file.write('server.document-root = "%s"\n' % http_server_doc_root)
 270   config_file.write('server.port = %d\n' % HTTP_PORT)
 271   # These lines are added so lighttpd_server.py's internal test succeeds.
 272   config_file.write('server.tag = "%s"\n' % http_server.server_tag)
 273   config_file.write('server.pid-file = "%s"\n' % http_server.pid_file)
 274   config_file.write('dir-listing.activate = "enable"\n')
 275   config_file.flush()
 276
 277
 278 def main():
 279   constants.SetBuildType(BUILD_TYPE)
 280   # Install APK
 281   device = GetDevice()
 282   device.EnableRoot()
 283   device.Install(APP_APK)
 284   # Start USB reverse tethering.
 285   # Port map is ignored for tethering; must create one to placate assertions.
 286   named_port_pair_map = {'http': (forwarders.PortPair(0, 0)),
 287       'https': None, 'dns': None}
 288   port_pairs = forwarders.PortPairs(**named_port_pair_map)
 289   forwarder = GetForwarderFactory(device).Create(port_pairs)
 290   # Start HTTP server.
 291   http_server_doc_root = GenerateHttpTestResources()
 292   config_file = tempfile.NamedTemporaryFile()
 293   http_server = lighttpd_server.LighttpdServer(http_server_doc_root,
 294       port=HTTP_PORT, base_config_path=config_file.name)
 295   GenerateLighttpdConfig(config_file, http_server_doc_root, http_server)
 296   assert http_server.StartupHttpServer()
 297   config_file.close()
 298   # Start QUIC server.
 299   quic_server_doc_root = GenerateQuicTestResources(device)
 300   quic_server = QuicServer(quic_server_doc_root)
 301   quic_server.StartupQuicServer(device)
 302   # Launch Telemetry's benchmark_runner on CronetPerfTestBenchmark.
 303   # By specifying this file's directory as the benchmark directory, it will
 304   # allow benchmark_runner to in turn open this file up and find the
 305   # CronetPerfTestBenchmark class to run the benchmark.
 306   top_level_dir = os.path.dirname(os.path.realpath(__file__))
 307   runner_config = benchmark_runner.ProjectConfig(
 308       top_level_dir=top_level_dir,
 309       benchmark_dirs=[top_level_dir])
 310   sys.argv.insert(1, 'run')
 311   sys.argv.insert(2, 'run.CronetPerfTestBenchmark')
 312   sys.argv.insert(3, '--android-rndis')
 313   benchmark_runner.main(runner_config)
 314   # Shutdown.
 315   quic_server.ShutdownQuicServer()
 316   shutil.rmtree(quic_server_doc_root)
 317   http_server.ShutdownHttpServer()
 318   shutil.rmtree(http_server_doc_root)
 319
 320
 321 if __name__ == '__main__':
 322   main()