testing/performance/talos/ttest.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is standalone Firefox Windows performance test.
  15 #
  16 # The Initial Developer of the Original Code is Google Inc.
  17 # Portions created by the Initial Developer are Copyright (C) 2006
  18 # the Initial Developer. All Rights Reserved.
  19 #
  20 # Contributor(s):
  21 #   Annie Sullivan <annie.sullivan@gmail.com> (original author)
  22 #   Ben Hearsum    <bhearsum@wittydomain.com> (OS independence)
  23 #   Alice Nodelman <anodelman@mozilla.com>
  24 #
  25 # Alternatively, the contents of this file may be used under the terms of
  26 # either the GNU General Public License Version 2 or later (the "GPL"), or
  27 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28 # in which case the provisions of the GPL or the LGPL are applicable instead
  29 # of those above. If you wish to allow use of your version of this file only
  30 # under the terms of either the GPL or the LGPL, and not to allow others to
  31 # use your version of this file under the terms of the MPL, indicate your
  32 # decision by deleting the provisions above and replace them with the notice
  33 # and other provisions required by the GPL or the LGPL. If you do not delete
  34 # the provisions above, a recipient may use your version of this file under
  35 # the terms of any one of the MPL, the GPL or the LGPL.
  36 #
  37 # ***** END LICENSE BLOCK *****
  38
  39 """A generic means of running an URL based browser test
  40    follows the following steps
  41      - creates a profile
  42      - tests the profile
  43      - gets metrics for the current test environment
  44      - loads the url
  45      - collects info on any counters while test runs
  46      - waits for a 'dump' from the browser
  47 """
  48
  49 __author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
  50
  51
  52 import platform
  53 import os
  54 import re
  55 import shutil
  56 import time
  57 import sys
  58 import subprocess
  59 import utils
  60 from utils import talosError
  61
  62 import ffprocess
  63 import ffsetup
  64
  65 if platform.system() == "Linux":
  66     from cmanager_linux import *
  67     platform_type = 'unix_'
  68 elif platform.system() in ("Windows", "Microsoft"):
  69     from cmanager_win32 import *
  70     platform_type = 'win_'
  71 elif platform.system() == "Darwin":
  72     from cmanager_mac import *
  73     platform_type = 'unix_'
  74
  75
  76 # Regula expression for getting results from most tests
  77 RESULTS_REGEX = re.compile('__start_report(.*)__end_report',
  78                       re.DOTALL | re.MULTILINE)
  79 # Regular expression to get stats for page load test (Tp) - should go away once data passing is standardized
  80 RESULTS_TP_REGEX = re.compile('__start_tp_report(.*)__end_tp_report',
  81                       re.DOTALL | re.MULTILINE)
  82 RESULTS_GENERIC = re.compile('(.*)', re.DOTALL | re.MULTILINE)
  83 RESULTS_REGEX_FAIL = re.compile('__FAIL(.*)__FAIL', re.DOTALL|re.MULTILINE)
  84
  85 def checkBrowserAlive():
  86   #is the browser actually up?
  87   return (ffprocess.ProcessesWithNameExist("firefox") and not ffprocess.ProcessesWithNameExist("crashreporter", "talkback", "dwwin"))
  88
  89 def checkAllProcesses():
  90   #is anything browser related active?
  91   return ffprocess.ProcessesWithNameExist("firefox", "crashreporter", "talkback", "dwwin")
  92
  93 def cleanupProcesses():
  94   #kill any remaining firefox processes
  95   ffprocess.TerminateAllProcesses("firefox", "crashreporter", "dwwin", "talkback")
  96   #check if anything is left behind
  97   if checkAllProcesses():
  98     #this is for windows machines.  when attempting to send kill messages to win processes the OS
  99     # always gives the process a chance to close cleanly before terminating it, this takes longer
 100     # and we need to give it a little extra time to complete
 101     time.sleep(10)
 102     if checkAllProcesses():
 103       raise talosError("failed to cleanup")
 104
 105 def createProfile(browser_config):
 106   if browser_config["profile_path"] != {}:
 107       # Create the new profile
 108       temp_dir, profile_dir = ffsetup.CreateTempProfileDir(browser_config['profile_path'],
 109                                                  browser_config['preferences'],
 110                                                  browser_config['extensions'])
 111       utils.debug("created profile")
 112   else:
 113       # no profile path was set in the config, set the profile_dir to an empty string.
 114       profile_dir = ""
 115   return profile_dir, temp_dir
 116
 117 def initializeProfile(profile_dir, browser_config):
 118   if browser_config["profile_path"] != {}:
 119       if not (ffsetup.InitializeNewProfile(browser_config['firefox'], profile_dir, browser_config['init_url'])):
 120          raise talosError("failed to initialize browser")
 121       time.sleep(10)
 122       if checkAllProcesses():
 123          raise talosError("browser failed to close after being initialized")
 124
 125 def cleanupProfile(dir, browser_config):
 126   # Delete the temp profile directory  Make it writeable first,
 127   # because every once in a while Firefox seems to drop a read-only
 128   # file into it.
 129   if browser_config["profile_path"] != {}:
 130     ffsetup.MakeDirectoryContentsWritable(dir)
 131     shutil.rmtree(dir)
 132
 133 def runTest(browser_config, test_config):
 134   """
 135   Runs an url based test on the browser as specified in the browser_config dictionary
 136
 137   Args:
 138     browser_config:  Dictionary of configuration options for the browser (paths, prefs, etc)
 139     test_config   :  Dictionary of configuration for the given test (url, cycles, counters, etc)
 140
 141   """
 142
 143   utils.debug("operating with platform_type : " + platform_type)
 144   counters = test_config[platform_type + 'counters']
 145   resolution = test_config['resolution']
 146   all_browser_results = []
 147   all_counter_results = []
 148   utils.setEnvironmentVars(browser_config['env'])
 149
 150
 151   try:
 152     if checkAllProcesses():
 153       utils.debug("firefox already running before testing started (unclean system)")
 154       raise talosError("system not clean")
 155
 156     # add any provided directories to the installed firefox
 157     for dir in browser_config['dirs']:
 158       ffsetup.InstallInBrowser(browser_config['firefox'], browser_config['dirs'][dir])
 159
 160     profile_dir, temp_dir = createProfile(browser_config)
 161     initializeProfile(profile_dir, browser_config)
 162
 163     utils.debug("initialized firefox")
 164     ffprocess.Sleep()
 165
 166     for i in range(test_config['cycles']):
 167       # check to see if the previous cycle is still hanging around
 168       if (i > 0) and checkAllProcesses():
 169         raise talosError("previous cycle still running")
 170       # Run the test
 171       browser_results = ""
 172       if 'timeout' in test_config:
 173         timeout = test_config['timeout']
 174       else:
 175         timeout = 28800 # 8 hours
 176       total_time = 0
 177       output = ''
 178       url = test_config['url']
 179       if 'url_mod' in test_config:
 180         url += eval(test_config['url_mod'])
 181       command_line = ffprocess.GenerateFirefoxCommandLine(browser_config['firefox'], profile_dir, url)
 182
 183       utils.debug("command line: " + command_line)
 184
 185       process = subprocess.Popen(command_line, stdout=subprocess.PIPE, universal_newlines=True, shell=True, bufsize=0, env=os.environ)
 186       handle = process.stdout
 187
 188       #give firefox a chance to open
 189       # this could mean that we are losing the first couple of data points as the tests starts, but if we don't provide
 190       # some time for the browser to start we have trouble connecting the CounterManager to it
 191       ffprocess.Sleep()
 192       #set up the counters for this test
 193       if counters:
 194         cm = CounterManager("firefox", counters)
 195         cm.startMonitor()
 196       counter_results = {}
 197       for counter in counters:
 198         counter_results[counter] = []
 199
 200       busted = False
 201       while total_time < timeout:
 202         # Sleep for [resolution] seconds
 203         time.sleep(resolution)
 204         total_time += resolution
 205
 206         # Get the output from all the possible counters
 207         for count_type in counters:
 208           val = cm.getCounterValue(count_type)
 209
 210           if (val):
 211             counter_results[count_type].append(val)
 212
 213         # Check to see if page load times were outputted
 214         (bytes, current_output) = ffprocess.NonBlockingReadProcessOutput(handle)
 215         output += current_output
 216         match = RESULTS_GENERIC.search(current_output)
 217         if match:
 218           if match.group(1):
 219             utils.noisy(match.group(1))
 220         match = RESULTS_REGEX.search(output)
 221         if match:
 222           browser_results += match.group(1)
 223           utils.debug("Matched basic results: " + browser_results)
 224           break
 225         #TODO: this a stop gap until all of the tests start outputting the same format
 226         match = RESULTS_TP_REGEX.search(output)
 227         if match:
 228           browser_results += match.group(1)
 229           utils.debug("Matched tp results: " + browser_results)
 230           break
 231         match = RESULTS_REGEX_FAIL.search(output)
 232         if match:
 233           browser_results += match.group(1)
 234           utils.debug("Matched fail results: " + browser_results)
 235           raise talosError(match.group(1))
 236
 237         #ensure that the browser is still running
 238         #check at intervals of 60 - this is just to cut down on load
 239         #use the busted check to ensure that we aren't catching a bad time slice where the browser has
 240         # completed the test and closed but we haven't picked up the result yet
 241         if busted:
 242           raise talosError("browser crash")
 243         if (total_time % 60 == 0):
 244           if not checkBrowserAlive():
 245             busted = True
 246
 247       if total_time >= timeout:
 248         raise talosError("timeout exceeded")
 249
 250       #stop the counter manager since this test is complete
 251       if counters:
 252         cm.stopMonitor()
 253
 254       utils.debug("Completed test with: " + browser_results)
 255
 256       all_browser_results.append(browser_results)
 257       all_counter_results.append(counter_results)
 258
 259     cleanupProcesses()
 260     cleanupProfile(temp_dir, browser_config)
 261
 262     utils.restoreEnvironmentVars()
 263
 264     return (all_browser_results, all_counter_results)
 265   except:
 266     try:
 267       cleanupProcesses()
 268       if vars().has_key('temp_dir'):
 269         cleanupProfile(temp_dir, browser_config)
 270     except talosError, te:
 271       utils.debug("cleanup error: " + te.msg)
 272     except:
 273       utils.debug("unknown error during cleanup")
 274     raise