testing/performance/talos/run_tests.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is standalone Firefox Windows performance test.
  15 #
  16 # The Initial Developer of the Original Code is Google Inc.
  17 # Portions created by the Initial Developer are Copyright (C) 2006
  18 # the Initial Developer. All Rights Reserved.
  19 #
  20 # Contributor(s):
  21 #   Annie Sullivan <annie.sullivan@gmail.com> (original author)
  22 #   Alice Nodelman <anodelman@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 """Runs extension performance tests.
  39
  40    This file runs Ts (startup time) and Tp (page load time) tests
  41    for an extension with different profiles.  It was originally
  42    written for the Google Toolbar for Firefox; to make it work for
  43    another extension modify get_xpi.py.  To change the preferences
  44    that are set on the profiles that are tested, edit the arrays in
  45    the main function below.
  46 """
  47
  48 __author__ = 'annie.sullivan@gmail.com (Annie Sullivan)'
  49
  50
  51 import time
  52 import yaml
  53 import sys
  54 import urllib
  55 import tempfile
  56 import os
  57 import string
  58 import socket
  59 socket.setdefaulttimeout(480)
  60 import getopt
  61
  62 import utils
  63 from utils import talosError
  64 import post_file
  65 import ttest
  66
  67 def shortNames(name):
  68   if name == "tp_loadtime":
  69     return "tp"
  70   elif name == "tp_js_loadtime":
  71     return "tp_js_l"
  72   elif name == "tp_Percent Processor Time":
  73     return "tp_%cpu"
  74   elif name == "tp_Working Set":
  75     return "tp_memset"
  76   elif name == "tp_Private Bytes":
  77     return "tp_pbytes"
  78   else:
  79     return name
  80
  81 def process_Request(post):
  82   str = ""
  83   lines = post.split('\n')
  84   for line in lines:
  85     if line.find("RETURN:") > -1:
  86         str += line.split(":")[3] + ":" + shortNames(line.split(":")[1]) + ":" + line.split(":")[2] + '\n'
  87     utils.debug("process_Request line: " + line.replace("RETURN", ""))
  88   return str
  89
  90 def send_to_csv(csv_dir, results):
  91   import csv
  92   for res in results:
  93     browser_dump, counter_dump = results[res]
  94     writer = csv.writer(open(os.path.join(csv_dir, res + '.csv'), "wb"))
  95     if res in ('ts', 'twinopen'):
  96       i = 0
  97       writer.writerow(['i', 'val'])
  98       for val in browser_dump:
  99         val_list = val.split('|')
 100         for v in val_list:
 101           writer.writerow([i, v])
 102           i += 1
 103     else:
 104       writer.writerow(['i', 'page', 'median', 'mean', 'min' , 'max', 'runs'])
 105       for bd in browser_dump:
 106         bd.rstrip('\n')
 107         page_results = bd.splitlines()
 108         i = 0
 109         for mypage in page_results:
 110           r = mypage.split(';')
 111           #skip this line if it isn't the correct format
 112           if len(r) == 1:
 113               continue
 114           r[1] = r[1].rstrip('/')
 115           if r[1].find('/') > -1 :
 116              page = r[1].split('/')[1]
 117           else:
 118              page = r[1]
 119           writer.writerow([i, page, r[2], r[3], r[4], r[5], '|'.join(r[6:])])
 120           i += 1
 121     for cd in counter_dump:
 122       for count_type in cd:
 123         writer = csv.writer(open(os.path.join(csv_dir, res + '_' + count_type + '.csv'), "wb"))
 124         writer.writerow(['i', 'value'])
 125         i = 0
 126         for val in cd[count_type]:
 127           writer.writerow([i, val])
 128           i += 1
 129
 130 def post_chunk(results_server, results_link, id, filename):
 131   tmpf = open(filename, "r")
 132   file_data = tmpf.read()
 133   try:
 134     ret = post_file.post_multipart(results_server, results_link, [("key", "value")], [("filename", filename, file_data)])
 135   except:
 136     print "FAIL: error in post data"
 137     sys.exit(0)
 138   links = process_Request(ret)
 139   utils.debug(id + ": sent results")
 140   return links
 141
 142 def chunk_list(val_list):
 143   """
 144     divide up a list into manageable chunks
 145     currently set at length 500
 146     this is for a failure on mac os x with python 2.4.4
 147   """
 148   chunks = []
 149   end = 500
 150   while (val_list != []):
 151     chunks.append(val_list[0:end])
 152     val_list = val_list[end:len(val_list)]
 153   return chunks
 154
 155 def send_to_graph(results_server, results_link, title, date, browser_config, results):
 156   tbox = title
 157   url_format = "http://%s/%s"
 158   link_format= "<a href=\"%s\">%s</a>"
 159   #value, testname, tbox, timeval, date, branch, buildid, type, data
 160   result_format = "%.2f,%s,%s,%d,%d,%s,%s,%s,%s,\n"
 161   result_format2 = "%.2f,%s,%s,%d,%d,%s,%s,%s,\n"
 162   links = ''
 163
 164   for res in results:
 165     browser_dump, counter_dump = results[res]
 166     utils.debug("Working with test: " + res)
 167     utils.debug("Sending results: " + " ".join(browser_dump))
 168     utils.stamped_msg("Transmitting test: " + res, "Started")
 169     filename = tempfile.mktemp()
 170     tmpf = open(filename, "w")
 171     if res in ('ts', 'twinopen'):
 172        i = 0
 173        for val in browser_dump:
 174         val_list = val.split('|')
 175         for v in val_list:
 176           tmpf.write(result_format % (float(v), res, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", "ms"))
 177           i += 1
 178     else:
 179       # each line of the string is of the format i;page_name;median;mean;min;max;time vals\n
 180       name = ''
 181       if ((res == 'tp') or (res == 'tp_js')):
 182           name = '_loadtime'
 183       for bd in browser_dump:
 184         bd.rstrip('\n')
 185         page_results = bd.splitlines()
 186         i = 0
 187         for mypage in page_results:
 188           r = mypage.split(';')
 189           #skip this line if it isn't the correct format
 190           if len(r) == 1:
 191               continue
 192           r[1] = r[1].rstrip('/')
 193           if r[1].find('/') > -1 :
 194              page = r[1].split('/')[1]
 195           else:
 196              page = r[1]
 197           try:
 198             val = float(r[2])
 199           except ValueError:
 200             print 'WARNING: value error for median in tp'
 201             val = 0
 202           tmpf.write(result_format % (val, res + name, tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete", page))
 203           i += 1
 204     tmpf.flush()
 205     tmpf.close()
 206     links += post_chunk(results_server, results_link, res, filename)
 207     os.remove(filename)
 208     for cd in counter_dump:
 209       for count_type in cd:
 210         val_list = cd[count_type]
 211         chunks = chunk_list(val_list)
 212         chunk_link = ''
 213         i = 0
 214         for chunk in chunks:
 215           filename = tempfile.mktemp()
 216           tmpf = open(filename, "w")
 217           for val in chunk:
 218               tmpf.write(result_format2 % (float(val), res + "_" + count_type.replace("%", "Percent"), tbox, i, date, browser_config['branch'], browser_config['buildid'], "discrete"))
 219               i += 1
 220           tmpf.flush()
 221           tmpf.close()
 222           chunk_link = post_chunk(results_server, results_link, '%s_%s (%d values)' % (res, count_type, len(chunk)), filename)
 223           os.remove(filename)
 224         links += chunk_link
 225     utils.stamped_msg("Transmitting test: " + res, "Stopped")
 226
 227   first_results = ''
 228   last_results = ''
 229   full_results = '\nRETURN:<p style="font-size:smaller;">Details:<br>'
 230   lines = links.split('\n')
 231   for line in lines:
 232     if line == "":
 233       continue
 234     values = line.split(":")
 235     linkName = values[1]
 236     if linkName in ('tp_pbytes', 'tp_%cpu'):
 237       continue
 238     if float(values[2]) > 0:
 239       linkName += ":&nbsp;" + str(values[2])
 240       url = url_format % (results_server, values[0])
 241       link = link_format % (url, linkName)
 242       first_results = first_results + "\nRETURN:" + link + '<br>'
 243     else:
 244       url = url_format % (results_server, values[0])
 245       link = link_format % (url, linkName)
 246       last_results = last_results + '| ' + link + ' '
 247   full_results = first_results + full_results + last_results + '|</p>'
 248   print full_results
 249
 250 def test_file(filename):
 251   """Runs the Ts and Tp tests on the given config file and generates a report.
 252
 253   Args:
 254     filename: the name of the file to run the tests on
 255   """
 256
 257   browser_config = []
 258   tests = []
 259   title = ''
 260   testdate = ''
 261   csv_dir = ''
 262   results_server = ''
 263   results_link = ''
 264   results = {}
 265
 266   # Read in the profile info from the YAML config file
 267   config_file = open(filename, 'r')
 268   yaml_config = yaml.load(config_file)
 269   config_file.close()
 270   for item in yaml_config:
 271     if item == 'title':
 272       title = yaml_config[item]
 273     elif item == 'testdate':
 274       testdate = yaml_config[item]
 275     elif item == 'csv_dir':
 276        csv_dir = os.path.normpath(yaml_config[item])
 277        if not os.path.exists(csv_dir):
 278          print "FAIL: path \"" + csv_dir + "\" does not exist"
 279          sys.exit(0)
 280     elif item == 'results_server':
 281        results_server = yaml_config[item]
 282     elif item == 'results_link' :
 283        results_link = yaml_config[item]
 284   if (results_link != results_server != ''):
 285     if not post_file.link_exists(results_server, results_link):
 286       sys.exit(0)
 287   browser_config = {'preferences'  : yaml_config['preferences'],
 288                     'extensions'   : yaml_config['extensions'],
 289                     'firefox'      : yaml_config['firefox'],
 290                     'branch'       : yaml_config['branch'],
 291                     'buildid'      : yaml_config['buildid'],
 292                     'profile_path' : yaml_config['profile_path'],
 293                     'env'          : yaml_config['env'],
 294                     'dirs'         : yaml_config['dirs'],
 295                     'init_url'     : yaml_config['init_url']}
 296   #normalize paths to work accross platforms
 297   browser_config['firefox'] = os.path.normpath(browser_config['firefox'])
 298   if browser_config['profile_path'] != {}:
 299     browser_config['profile_path'] = os.path.normpath(browser_config['profile_path'])
 300   for dir in browser_config['dirs']:
 301     browser_config['dirs'][dir] = os.path.normpath(browser_config['dirs'][dir])
 302   tests = yaml_config['tests']
 303   config_file.close()
 304   if (testdate != ''):
 305     date = int(time.mktime(time.strptime(testdate, '%a, %d %b %Y %H:%M:%S GMT')))
 306   else:
 307     date = int(time.time()) #TODO get this into own file
 308   utils.debug("using testdate: %d" % date)
 309   utils.debug("actual date: %d" % int(time.time()))
 310
 311   utils.stamped_msg(title, "Started")
 312   for test in tests:
 313     utils.stamped_msg("Running test " + test, "Started")
 314     try:
 315       browser_dump, counter_dump = ttest.runTest(browser_config, tests[test])
 316     except talosError, e:
 317       utils.stamped_msg("Failed " + test, "Stopped")
 318       print 'FAIL: Busted: ' + test
 319       print 'FAIL: ' + e.msg
 320       sys.exit(0)
 321     utils.debug("Received test results: " + " ".join(browser_dump))
 322     results[test] = [browser_dump, counter_dump]
 323     utils.stamped_msg("Completed test " + test, "Stopped")
 324   utils.stamped_msg(title, "Stopped")
 325
 326   #process the results
 327   if (results_server != '') and (results_link != ''):
 328     #send results to the graph server
 329     utils.stamped_msg("Sending results", "Started")
 330     send_to_graph(results_server, results_link, title, date, browser_config, results)
 331     utils.stamped_msg("Completed sending results", "Stopped")
 332   if csv_dir != '':
 333     send_to_csv(csv_dir, results)
 334
 335 if __name__=='__main__':
 336   optlist, args = getopt.getopt(sys.argv[1:], 'dn', ['debug', 'noisy'])
 337   for o, a in optlist:
 338     if o in ('-d', "--debug"):
 339       print 'setting debug'
 340       utils.setdebug(1)
 341     if o in ('-n', "--noisy"):
 342       utils.setnoisy(1)
 343   # Read in each config file and run the tests on it.
 344   for arg in args:
 345     utils.debug("running test file " + arg)
 346     test_file(arg)
 347