testing/performance/win32/tp.py

   1 #!c:/Python24/python.exe
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is standalone Firefox Windows performance test.
  17 #
  18 # The Initial Developer of the Original Code is Google Inc.
  19 # Portions created by the Initial Developer are Copyright (C) 2006
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Annie Sullivan <annie.sullivan@gmail.com> (original author)
  24 #
  25 # Alternatively, the contents of this file may be used under the terms of
  26 # either the GNU General Public License Version 2 or later (the "GPL"), or
  27 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  28 # in which case the provisions of the GPL or the LGPL are applicable instead
  29 # of those above. If you wish to allow use of your version of this file only
  30 # under the terms of either the GPL or the LGPL, and not to allow others to
  31 # use your version of this file under the terms of the MPL, indicate your
  32 # decision by deleting the provisions above and replace them with the notice
  33 # and other provisions required by the GPL or the LGPL. If you do not delete
  34 # the provisions above, a recipient may use your version of this file under
  35 # the terms of any one of the MPL, the GPL or the LGPL.
  36 #
  37 # ***** END LICENSE BLOCK *****
  38
  39 """A set of functions to run the Tp test.
  40
  41    The Tp test measures page load times in Firefox.  It does this with a
  42    JavaScript script that opens a new window and cycles through page loads
  43    from the local disk, timing each one.  The script dumps the sum of the
  44    mean times to open each page, and the standard deviation, to standard out.
  45    We can also measure performance attributes during the test.  See
  46    http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
  47    for a description of what can be measured.
  48 """
  49
  50 __author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
  51
  52
  53 import os
  54 import re
  55 import shutil
  56 import time
  57 import win32pdh
  58 import win32pdhutil
  59
  60 import ffprocess
  61 import ffprofile
  62 import paths
  63
  64
  65 # Regular expression to get stats for page load test (Tp)
  66 TP_REGEX = re.compile('__start_page_load_report(.*)__end_page_load_report',
  67                       re.DOTALL | re.MULTILINE)
  68
  69
  70 def AddCounter(counter_name):
  71   """Adds a pdh query and counter of the given name to Firefox.
  72
  73   Args: counter_name: The name of the counter to add, i.e. "% Processor Time"
  74
  75   Returns:
  76     (query handle, counter handle)
  77   """
  78
  79   path = win32pdh.MakeCounterPath( (None,
  80                                     'process',
  81                                     'firefox',
  82                                     None,
  83                                     -1,
  84                                     counter_name) )
  85   hq = win32pdh.OpenQuery()
  86   try:
  87     hc = win32pdh.AddCounter(hq, path)
  88   except:
  89     win32pdh.CloseQuery(hq)
  90   return hq, hc
  91
  92
  93 def CleanupCounter(hq, hc):
  94   """Cleans up a counter after it is no longer needed.
  95
  96   Args:
  97     hq: handle to the query for the counter
  98     hc: handle to the counter
  99   """
 100
 101   try:
 102     win32pdh.RemoveCounter(hc)
 103     win32pdh.CloseQuery(hq)
 104   except:
 105     # Sometimes we get unexpected win32 errors.  Not much can be done.
 106     pass
 107
 108
 109 def GetCounterValue(hq, hc):
 110   """Returns the current value of the given counter
 111
 112   Args:
 113     hq: Handle of the query for the counter
 114     hc: Handle of the counter
 115
 116   Returns:
 117     The current value of the counter
 118   """
 119
 120   try:
 121     win32pdh.CollectQueryData(hq)
 122     type, val = win32pdh.GetFormattedCounterValue(hc, win32pdh.PDH_FMT_LONG)
 123     return val
 124   except:
 125     return None
 126
 127
 128 def RunPltTests(source_profile_dir,
 129                 profile_configs,
 130                 num_cycles,
 131                 counters,
 132                 resolution):
 133   """Runs the Page Load tests with profiles created from the given
 134      base diectory and list of configuations.
 135
 136   Args:
 137     source_profile_dir:  Full path to base directory to copy profile from.
 138     profile_configs:  Array of configuration options for each profile.
 139       These are of the format:
 140       [{prefname:prevalue,prefname2:prefvalue2},
 141        {extensionguid:path_to_extension}],[{prefname...
 142     num_cycles: Number of times to cycle through all the urls on each test.
 143     counters: Array of counters ("% Processor Time", "Working Set", etc)
 144               See http://technet2.microsoft.com/WindowsServer/en/Library/86b5d116-6fb3-427b-af8c-9077162125fe1033.mspx?mfr=true
 145               for a list of available counters and their descriptions.
 146     resolution: Time (in seconds) between collecting counters
 147
 148   Returns:
 149     A tuple containing:
 150       An array of plt results for each run.  For example:
 151         ["mean: 150.30\nstdd:34.2", "mean 170.33\nstdd:22.4"]
 152       An array of counter data from each run.  For example:
 153         [{"counter1": [1, 2, 3], "counter2":[4,5,6]},
 154          {"counter1":[1,3,5], "counter2":[2,4,6]}]
 155   """
 156
 157   counter_data = []
 158   plt_results = []
 159   for config in profile_configs:
 160     # Create the new profile
 161     profile_dir = ffprofile.CreateTempProfileDir(source_profile_dir,
 162                                                  config[0],
 163                                                  config[1])
 164
 165     # Run Firefox once with new profile so initializing it doesn't cause
 166     # a performance hit, and the second Firefox that gets created is properly
 167     # terminated.
 168     ffprofile.InitializeNewProfile(config[2], profile_dir)
 169     ffprocess.SyncAndSleep()
 170
 171     # Run the plt test for this profile
 172     timeout = 300
 173     total_time = 0
 174     output = ''
 175     url = paths.TP_URL + '?cycles=' + str(num_cycles)
 176     command_line = ffprocess.GenerateFirefoxCommandLine(config[2], profile_dir, url)
 177     handle = os.popen(command_line)
 178     # PDH might need to be "refreshed" if it has been queried while
 179     # Firefox is closed.
 180     win32pdh.EnumObjects(None, None, 0, 1)
 181
 182     # Initialize counts
 183     counts = {}
 184     counter_handles = {}
 185     for counter in counters:
 186       counts[counter] = []
 187       counter_handles[counter] = AddCounter(counter)
 188
 189     while total_time < timeout:
 190
 191       # Sleep for [resolution] seconds
 192       time.sleep(resolution)
 193       total_time += resolution
 194
 195       # Get the output from all the possible counters
 196       for count_type in counters:
 197         val = GetCounterValue(counter_handles[count_type][0],
 198                               counter_handles[count_type][1])
 199         if (val):
 200           # Sometimes the first sample can be None, or PLT test will have
 201           # closed Firefox by this point.  Only count real values.
 202           counts[count_type].append(val)
 203
 204       # Check to see if page load times were outputted
 205       (bytes, current_output) = ffprocess.NonBlockingReadProcessOutput(handle)
 206       output += current_output
 207       match = TP_REGEX.search(output)
 208       if match:
 209         plt_results.append(match.group(1))
 210         break;
 211
 212     # Cleanup counters
 213     for counter in counters:
 214       CleanupCounter(counter_handles[counter][0], counter_handles[counter][1])
 215
 216     # Firefox should have exited cleanly, but close it if it doesn't
 217     # after 2 seconds.
 218     time.sleep(2)
 219     ffprocess.TerminateAllProcesses("firefox")
 220     ffprocess.SyncAndSleep()
 221
 222     # Delete the temp profile directory  Make it writeable first,
 223     # because every once in a while Firefox seems to drop a read-only
 224     # file into it.
 225     ffprofile.MakeDirectoryContentsWritable(profile_dir)
 226     shutil.rmtree(profile_dir)
 227
 228     counter_data.append(counts)
 229
 230   return (plt_results, counter_data)