tools/telemetry/third_party/webpagereplay/replay.py

   1 #!/usr/bin/env python
   2 # Copyright 2010 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #      http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15
  16 """Replays web pages under simulated network conditions.
  17
  18 Must be run as administrator (sudo).
  19
  20 To record web pages:
  21   1. Start the program in record mode.
  22      $ sudo ./replay.py --record archive.wpr
  23   2. Load the web pages you want to record in a web browser. It is important to
  24      clear browser caches before this so that all subresources are requested
  25      from the network.
  26   3. Kill the process to stop recording.
  27
  28 To replay web pages:
  29   1. Start the program in replay mode with a previously recorded archive.
  30      $ sudo ./replay.py archive.wpr
  31   2. Load recorded pages in a web browser. A 404 will be served for any pages or
  32      resources not in the recorded archive.
  33
  34 Network simulation examples:
  35   # 128KByte/s uplink bandwidth, 4Mbps/s downlink bandwidth with 100ms RTT time
  36   $ sudo ./replay.py --up 128KByte/s --down 4Mbit/s --delay_ms=100 archive.wpr
  37
  38   # 1% packet loss rate
  39   $ sudo ./replay.py --packet_loss_rate=0.01 archive.wpr
  40 """
  41
  42 import json
  43 import logging
  44 import optparse
  45 import os
  46 import socket
  47 import sys
  48 import traceback
  49
  50 import customhandlers
  51 import dnsproxy
  52 import httparchive
  53 import httpclient
  54 import httpproxy
  55 import net_configs
  56 import platformsettings
  57 import rules_parser
  58 import script_injector
  59 import servermanager
  60 import trafficshaper
  61
  62 if sys.version < '2.6':
  63   print 'Need Python 2.6 or greater.'
  64   sys.exit(1)
  65
  66
  67 def configure_logging(log_level_name, log_file_name=None):
  68   """Configure logging level and format.
  69
  70   Args:
  71     log_level_name: 'debug', 'info', 'warning', 'error', or 'critical'.
  72     log_file_name: a file name
  73   """
  74   if logging.root.handlers:
  75     logging.critical('A logging method (e.g. "logging.warn(...)")'
  76                      ' was called before logging was configured.')
  77   log_level = getattr(logging, log_level_name.upper())
  78   log_format = '%(asctime)s %(levelname)s %(message)s'
  79   logging.basicConfig(level=log_level, format=log_format)
  80   logger = logging.getLogger()
  81   if log_file_name:
  82     fh = logging.FileHandler(log_file_name)
  83     fh.setLevel(log_level)
  84     fh.setFormatter(logging.Formatter(log_format))
  85     logger.addHandler(fh)
  86   system_handler = platformsettings.get_system_logging_handler()
  87   if system_handler:
  88     logger.addHandler(system_handler)
  89
  90
  91 def AddDnsForward(server_manager, host):
  92   """Forward DNS traffic."""
  93   server_manager.Append(platformsettings.set_temporary_primary_nameserver, host)
  94
  95
  96 def AddDnsProxy(server_manager, options, host, port, real_dns_lookup,
  97                 http_archive):
  98   dns_filters = []
  99   if options.dns_private_passthrough:
 100     private_filter = dnsproxy.PrivateIpFilter(real_dns_lookup, http_archive)
 101     dns_filters.append(private_filter)
 102     server_manager.AppendRecordCallback(private_filter.InitializeArchiveHosts)
 103     server_manager.AppendReplayCallback(private_filter.InitializeArchiveHosts)
 104   if options.shaping_dns:
 105     delay_filter = dnsproxy.DelayFilter(options.record, **options.shaping_dns)
 106     dns_filters.append(delay_filter)
 107     server_manager.AppendRecordCallback(delay_filter.SetRecordMode)
 108     server_manager.AppendReplayCallback(delay_filter.SetReplayMode)
 109   server_manager.Append(dnsproxy.DnsProxyServer, host, port,
 110                         dns_lookup=dnsproxy.ReplayDnsLookup(host, dns_filters))
 111
 112
 113 def AddWebProxy(server_manager, options, host, real_dns_lookup, http_archive):
 114   if options.rules_path:
 115     with open(options.rules_path) as file_obj:
 116       allowed_imports = [
 117           name.strip() for name in options.allowed_rule_imports.split(',')]
 118       rules = rules_parser.Rules(file_obj, allowed_imports)
 119     logging.info('Parsed %s rules:\n%s', options.rules_path, rules)
 120   else:
 121     rules = rules_parser.Rules()
 122   inject_script = script_injector.GetInjectScript(options.inject_scripts)
 123   custom_handlers = customhandlers.CustomHandlers(options, http_archive)
 124   custom_handlers.add_server_manager_handler(server_manager)
 125   archive_fetch = httpclient.ControllableHttpArchiveFetch(
 126       http_archive, real_dns_lookup,
 127       inject_script,
 128       options.diff_unknown_requests, options.record,
 129       use_closest_match=options.use_closest_match,
 130       scramble_images=options.scramble_images)
 131   server_manager.AppendRecordCallback(archive_fetch.SetRecordMode)
 132   server_manager.AppendReplayCallback(archive_fetch.SetReplayMode)
 133   server_manager.Append(
 134       httpproxy.HttpProxyServer,
 135       archive_fetch, custom_handlers, rules,
 136       host=host, port=options.port, use_delays=options.use_server_delay,
 137       **options.shaping_http)
 138   if options.ssl:
 139     if options.should_generate_certs:
 140       server_manager.Append(
 141           httpproxy.HttpsProxyServer, archive_fetch, custom_handlers, rules,
 142           options.https_root_ca_cert_path, host=host, port=options.ssl_port,
 143           use_delays=options.use_server_delay, **options.shaping_http)
 144     else:
 145       server_manager.Append(
 146           httpproxy.SingleCertHttpsProxyServer, archive_fetch,
 147           custom_handlers, rules, options.https_root_ca_cert_path, host=host,
 148           port=options.ssl_port, use_delays=options.use_server_delay,
 149           **options.shaping_http)
 150   if options.http_to_https_port:
 151     server_manager.Append(
 152         httpproxy.HttpToHttpsProxyServer,
 153         archive_fetch, custom_handlers, rules,
 154         host=host, port=options.http_to_https_port,
 155         use_delays=options.use_server_delay,
 156         **options.shaping_http)
 157
 158
 159 def AddTrafficShaper(server_manager, options, host):
 160   if options.shaping_dummynet:
 161     server_manager.AppendTrafficShaper(
 162         trafficshaper.TrafficShaper, host=host,
 163         use_loopback=not options.server_mode and host == '127.0.0.1',
 164         **options.shaping_dummynet)
 165
 166
 167 class OptionsWrapper(object):
 168   """Add checks, updates, and methods to option values.
 169
 170   Example:
 171     options, args = option_parser.parse_args()
 172     options = OptionsWrapper(options, option_parser)  # run checks and updates
 173     if options.record and options.HasTrafficShaping():
 174        [...]
 175   """
 176   _TRAFFICSHAPING_OPTIONS = {
 177       'down', 'up', 'delay_ms', 'packet_loss_rate', 'init_cwnd', 'net'}
 178   _CONFLICTING_OPTIONS = (
 179       ('record', ('down', 'up', 'delay_ms', 'packet_loss_rate', 'net',
 180                   'spdy', 'use_server_delay')),
 181       ('append', ('down', 'up', 'delay_ms', 'packet_loss_rate', 'net',
 182                   'use_server_delay')),  # same as --record
 183       ('net', ('down', 'up', 'delay_ms')),
 184       ('server', ('server_mode',)),
 185   )
 186
 187   def __init__(self, options, parser):
 188     self._options = options
 189     self._parser = parser
 190     self._nondefaults = set([
 191         name for name, value in parser.defaults.items()
 192         if getattr(options, name) != value])
 193     self._CheckConflicts()
 194     self._CheckValidIp('host')
 195     self._CheckFeatureSupport()
 196     self._MassageValues()
 197
 198   def _CheckConflicts(self):
 199     """Give an error if mutually exclusive options are used."""
 200     for option, bad_options in self._CONFLICTING_OPTIONS:
 201       if option in self._nondefaults:
 202         for bad_option in bad_options:
 203           if bad_option in self._nondefaults:
 204             self._parser.error('Option --%s cannot be used with --%s.' %
 205                                 (bad_option, option))
 206
 207   def _CheckValidIp(self, name):
 208     """Give an error if option |name| is not a valid IPv4 address."""
 209     value = getattr(self._options, name)
 210     if value:
 211       try:
 212         socket.inet_aton(value)
 213       except Exception:
 214         self._parser.error('Option --%s must be a valid IPv4 address.' % name)
 215
 216   def _CheckFeatureSupport(self):
 217     if (self._options.should_generate_certs and
 218         not platformsettings.HasSniSupport()):
 219       self._parser.error('Option --should_generate_certs requires pyOpenSSL '
 220                          '0.13 or greater for SNI support.')
 221
 222   def _ShapingKeywordArgs(self, shaping_key):
 223     """Return the shaping keyword args for |shaping_key|.
 224
 225     Args:
 226       shaping_key: one of 'dummynet', 'dns', 'http'.
 227     Returns:
 228       {}  # if shaping_key does not apply, or options have default values.
 229       {k: v, ...}
 230     """
 231     kwargs = {}
 232     def AddItemIfSet(d, kw_key, opt_key=None):
 233       opt_key = opt_key or kw_key
 234       if opt_key in self._nondefaults:
 235         d[kw_key] = getattr(self, opt_key)
 236     if ((self.shaping_type == 'proxy' and shaping_key in ('dns', 'http')) or
 237         self.shaping_type == shaping_key):
 238       AddItemIfSet(kwargs, 'delay_ms')
 239       if shaping_key in ('dummynet', 'http'):
 240         AddItemIfSet(kwargs, 'down_bandwidth', opt_key='down')
 241         AddItemIfSet(kwargs, 'up_bandwidth', opt_key='up')
 242         if shaping_key == 'dummynet':
 243           AddItemIfSet(kwargs, 'packet_loss_rate')
 244           AddItemIfSet(kwargs, 'init_cwnd')
 245         elif self.shaping_type != 'none':
 246           if 'packet_loss_rate' in self._nondefaults:
 247             logging.warn('Shaping type, %s, ignores --packet_loss_rate=%s',
 248                          self.shaping_type, self.packet_loss_rate)
 249           if 'init_cwnd' in self._nondefaults:
 250             logging.warn('Shaping type, %s, ignores --init_cwnd=%s',
 251                          self.shaping_type, self.init_cwnd)
 252     return kwargs
 253
 254   def _MassageValues(self):
 255     """Set options that depend on the values of other options."""
 256     if self.append and not self.record:
 257       self._options.record = True
 258     if self.net:
 259       self._options.down, self._options.up, self._options.delay_ms = \
 260           net_configs.GetNetConfig(self.net)
 261       self._nondefaults.update(['down', 'up', 'delay_ms'])
 262     if not self.ssl:
 263       self._options.https_root_ca_cert_path = None
 264     self.shaping_dns = self._ShapingKeywordArgs('dns')
 265     self.shaping_http = self._ShapingKeywordArgs('http')
 266     self.shaping_dummynet = self._ShapingKeywordArgs('dummynet')
 267
 268   def __getattr__(self, name):
 269     """Make the original option values available."""
 270     return getattr(self._options, name)
 271
 272   def __repr__(self):
 273     """Return a json representation of the original options dictionary."""
 274     return json.dumps(self._options.__dict__)
 275
 276   def IsRootRequired(self):
 277     """Returns True iff the options require whole program root access."""
 278     if self.server:
 279       return True
 280
 281     def IsPrivilegedPort(port):
 282       return port and port < 1024
 283
 284     if IsPrivilegedPort(self.port) or (self.ssl and
 285                                        IsPrivilegedPort(self.ssl_port)):
 286       return True
 287
 288     if self.dns_forwarding:
 289       if IsPrivilegedPort(self.dns_port):
 290         return True
 291       if not self.server_mode and self.host == '127.0.0.1':
 292         return True
 293
 294     return False
 295
 296
 297 def replay(options, replay_filename):
 298   if options.admin_check and options.IsRootRequired():
 299     platformsettings.rerun_as_administrator()
 300   configure_logging(options.log_level, options.log_file)
 301   server_manager = servermanager.ServerManager(options.record)
 302   if options.server:
 303     AddDnsForward(server_manager, options.server)
 304   else:
 305     real_dns_lookup = dnsproxy.RealDnsLookup(
 306         name_servers=[platformsettings.get_original_primary_nameserver()])
 307     if options.record:
 308       httparchive.HttpArchive.AssertWritable(replay_filename)
 309       if options.append and os.path.exists(replay_filename):
 310         http_archive = httparchive.HttpArchive.Load(replay_filename)
 311         logging.info('Appending to %s (loaded %d existing responses)',
 312                      replay_filename, len(http_archive))
 313       else:
 314         http_archive = httparchive.HttpArchive()
 315     else:
 316       http_archive = httparchive.HttpArchive.Load(replay_filename)
 317       logging.info('Loaded %d responses from %s',
 318                    len(http_archive), replay_filename)
 319     server_manager.AppendRecordCallback(real_dns_lookup.ClearCache)
 320     server_manager.AppendRecordCallback(http_archive.clear)
 321
 322     ipfw_dns_host = None
 323     if options.dns_forwarding or options.shaping_dummynet:
 324       # compute the ip/host used for the DNS server and traffic shaping
 325       ipfw_dns_host = options.host
 326       if not ipfw_dns_host:
 327         ipfw_dns_host = platformsettings.get_server_ip_address(
 328             options.server_mode)
 329
 330     if options.dns_forwarding:
 331       if not options.server_mode and ipfw_dns_host == '127.0.0.1':
 332         AddDnsForward(server_manager, ipfw_dns_host)
 333       AddDnsProxy(server_manager, options, ipfw_dns_host, options.dns_port,
 334                   real_dns_lookup, http_archive)
 335     if options.ssl and options.https_root_ca_cert_path is None:
 336       options.https_root_ca_cert_path = os.path.join(os.path.dirname(__file__),
 337                                                      'wpr_cert.pem')
 338     http_proxy_address = options.host
 339     if not http_proxy_address:
 340       http_proxy_address = platformsettings.get_httpproxy_ip_address(
 341           options.server_mode)
 342     AddWebProxy(server_manager, options, http_proxy_address, real_dns_lookup,
 343                 http_archive)
 344     AddTrafficShaper(server_manager, options, ipfw_dns_host)
 345
 346   exit_status = 0
 347   try:
 348     server_manager.Run()
 349   except KeyboardInterrupt:
 350     logging.info('Shutting down.')
 351   except (dnsproxy.DnsProxyException,
 352           trafficshaper.TrafficShaperException,
 353           platformsettings.NotAdministratorError,
 354           platformsettings.DnsUpdateError) as e:
 355     logging.critical('%s: %s', e.__class__.__name__, e)
 356     exit_status = 1
 357   except Exception:
 358     logging.critical(traceback.format_exc())
 359     exit_status = 2
 360
 361   if options.record:
 362     http_archive.Persist(replay_filename)
 363     logging.info('Saved %d responses to %s', len(http_archive), replay_filename)
 364   return exit_status
 365
 366
 367 def GetOptionParser():
 368   class PlainHelpFormatter(optparse.IndentedHelpFormatter):
 369     def format_description(self, description):
 370       if description:
 371         return description + '\n'
 372       else:
 373         return ''
 374   option_parser = optparse.OptionParser(
 375       usage='%prog [options] replay_file',
 376       formatter=PlainHelpFormatter(),
 377       description=__doc__,
 378       epilog='http://code.google.com/p/web-page-replay/')
 379
 380   option_parser.add_option('-r', '--record', default=False,
 381       action='store_true',
 382       help='Download real responses and record them to replay_file')
 383   option_parser.add_option('--append', default=False,
 384       action='store_true',
 385       help='Append responses to replay_file.')
 386   option_parser.add_option('-l', '--log_level', default='debug',
 387       action='store',
 388       type='choice',
 389       choices=('debug', 'info', 'warning', 'error', 'critical'),
 390       help='Minimum verbosity level to log')
 391   option_parser.add_option('-f', '--log_file', default=None,
 392       action='store',
 393       type='string',
 394       help='Log file to use in addition to writting logs to stderr.')
 395
 396   network_group = optparse.OptionGroup(option_parser,
 397       'Network Simulation Options',
 398       'These options configure the network simulation in replay mode')
 399   network_group.add_option('-u', '--up', default='0',
 400       action='store',
 401       type='string',
 402       help='Upload Bandwidth in [K|M]{bit/s|Byte/s}. Zero means unlimited.')
 403   network_group.add_option('-d', '--down', default='0',
 404       action='store',
 405       type='string',
 406       help='Download Bandwidth in [K|M]{bit/s|Byte/s}. Zero means unlimited.')
 407   network_group.add_option('-m', '--delay_ms', default='0',
 408       action='store',
 409       type='string',
 410       help='Propagation delay (latency) in milliseconds. Zero means no delay.')
 411   network_group.add_option('-p', '--packet_loss_rate', default='0',
 412       action='store',
 413       type='string',
 414       help='Packet loss rate in range [0..1]. Zero means no loss.')
 415   network_group.add_option('-w', '--init_cwnd', default='0',
 416       action='store',
 417       type='string',
 418       help='Set initial cwnd (linux only, requires kernel patch)')
 419   network_group.add_option('--net', default=None,
 420       action='store',
 421       type='choice',
 422       choices=net_configs.NET_CONFIG_NAMES,
 423       help='Select a set of network options: %s.' % ', '.join(
 424           net_configs.NET_CONFIG_NAMES))
 425   network_group.add_option('--shaping_type', default='dummynet',
 426       action='store',
 427       choices=('dummynet', 'proxy'),
 428       help='When shaping is configured (i.e. --up, --down, etc.) decides '
 429            'whether to use |dummynet| (default), or |proxy| servers.')
 430   option_parser.add_option_group(network_group)
 431
 432   harness_group = optparse.OptionGroup(option_parser,
 433       'Replay Harness Options',
 434       'These advanced options configure various aspects of the replay harness')
 435   harness_group.add_option('-S', '--server', default=None,
 436       action='store',
 437       type='string',
 438       help='IP address of host running "replay.py --server_mode". '
 439            'This only changes the primary DNS nameserver to use the given IP.')
 440   harness_group.add_option('-M', '--server_mode', default=False,
 441       action='store_true',
 442       help='Run replay DNS & http proxies, and trafficshaping on --port '
 443            'without changing the primary DNS nameserver. '
 444            'Other hosts may connect to this using "replay.py --server" '
 445            'or by pointing their DNS to this server.')
 446   harness_group.add_option('-i', '--inject_scripts', default='deterministic.js',
 447       action='store',
 448       dest='inject_scripts',
 449       help='A comma separated list of JavaScript sources to inject in all '
 450            'pages. By default a script is injected that eliminates sources '
 451            'of entropy such as Date() and Math.random() deterministic. '
 452            'CAUTION: Without deterministic.js, many pages will not replay.')
 453   harness_group.add_option('-D', '--no-diff_unknown_requests', default=True,
 454       action='store_false',
 455       dest='diff_unknown_requests',
 456       help='During replay, do not show a diff of unknown requests against '
 457            'their nearest match in the archive.')
 458   harness_group.add_option('-C', '--use_closest_match', default=False,
 459       action='store_true',
 460       dest='use_closest_match',
 461       help='During replay, if a request is not found, serve the closest match'
 462            'in the archive instead of giving a 404.')
 463   harness_group.add_option('-U', '--use_server_delay', default=False,
 464       action='store_true',
 465       dest='use_server_delay',
 466       help='During replay, simulate server delay by delaying response time to'
 467            'requests.')
 468   harness_group.add_option('-I', '--screenshot_dir', default=None,
 469       action='store',
 470       type='string',
 471       help='Save PNG images of the loaded page in the given directory.')
 472   harness_group.add_option('-P', '--no-dns_private_passthrough', default=True,
 473       action='store_false',
 474       dest='dns_private_passthrough',
 475       help='Don\'t forward DNS requests that resolve to private network '
 476            'addresses. CAUTION: With this option important services like '
 477            'Kerberos will resolve to the HTTP proxy address.')
 478   harness_group.add_option('-x', '--no-dns_forwarding', default=True,
 479       action='store_false',
 480       dest='dns_forwarding',
 481       help='Don\'t forward DNS requests to the local replay server. '
 482            'CAUTION: With this option an external mechanism must be used to '
 483            'forward traffic to the replay server.')
 484   harness_group.add_option('--host', default=None,
 485       action='store',
 486       type='str',
 487       help='The IP address to bind all servers to. Defaults to 0.0.0.0 or '
 488            '127.0.0.1, depending on --server_mode and platform.')
 489   harness_group.add_option('-o', '--port', default=80,
 490       action='store',
 491       type='int',
 492       help='Port number to listen on.')
 493   harness_group.add_option('--ssl_port', default=443,
 494       action='store',
 495       type='int',
 496       help='SSL port number to listen on.')
 497   harness_group.add_option('--http_to_https_port', default=None,
 498       action='store',
 499       type='int',
 500       help='Port on which WPR will listen for HTTP requests that it will send '
 501            'along as HTTPS requests.')
 502   harness_group.add_option('--dns_port', default=53,
 503       action='store',
 504       type='int',
 505       help='DNS port number to listen on.')
 506   harness_group.add_option('-c', '--https_root_ca_cert_path', default=None,
 507       action='store',
 508       type='string',
 509       help='Certificate file to use with SSL (gets auto-generated if needed).')
 510   harness_group.add_option('--no-ssl', default=True,
 511       action='store_false',
 512       dest='ssl',
 513       help='Do not setup an SSL proxy.')
 514   option_parser.add_option_group(harness_group)
 515   harness_group.add_option('--should_generate_certs', default=False,
 516       action='store_true',
 517       help='Use OpenSSL to generate certificate files for requested hosts.')
 518   harness_group.add_option('--no-admin-check', default=True,
 519       action='store_false',
 520       dest='admin_check',
 521       help='Do not check if administrator access is needed.')
 522   harness_group.add_option('--scramble_images', default=False,
 523       action='store_true',
 524       dest='scramble_images',
 525       help='Scramble image responses.')
 526   harness_group.add_option('--rules_path', default=None,
 527       action='store',
 528       help='Path of file containing Python rules.')
 529   harness_group.add_option('--allowed_rule_imports', default='rules',
 530       action='store',
 531       help='A comma-separate list of allowed rule imports, or \'*\' to allow'
 532            ' all packages.  Defaults to \'%default\'.')
 533   return option_parser
 534
 535
 536 def main():
 537   option_parser = GetOptionParser()
 538   options, args = option_parser.parse_args()
 539   options = OptionsWrapper(options, option_parser)
 540
 541   if options.server:
 542     replay_filename = None
 543   elif len(args) != 1:
 544     option_parser.error('Must specify a replay_file')
 545   else:
 546     replay_filename = args[0]
 547
 548   return replay(options, replay_filename)
 549
 550
 551 if __name__ == '__main__':
 552   sys.exit(main())