testing/mochitest/runtests.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 1998
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 """
  41 Runs the Mochitest test harness.
  42 """
  43
  44 from datetime import datetime
  45 import logging
  46 import optparse
  47 import os
  48 import os.path
  49 import re
  50 import sys
  51 import time
  52 from urllib import quote_plus as encodeURIComponent
  53 import urllib2
  54 import commands
  55 import automation
  56
  57 # Path to the test script on the server
  58 TEST_SERVER_HOST = "localhost:8888"
  59 TEST_PATH = "/tests/"
  60 CHROME_PATH = "/redirect.html";
  61 A11Y_PATH = "/redirect-a11y.html"
  62 TESTS_URL = "http://" + TEST_SERVER_HOST + TEST_PATH
  63 CHROMETESTS_URL = "http://" + TEST_SERVER_HOST + CHROME_PATH
  64 A11YTESTS_URL = "http://" + TEST_SERVER_HOST + A11Y_PATH
  65 SERVER_SHUTDOWN_URL = "http://" + TEST_SERVER_HOST + "/server/shutdown"
  66 # main browser chrome URL, same as browser.chromeURL pref
  67 #ifdef MOZ_SUITE
  68 BROWSER_CHROME_URL = "chrome://navigator/content/navigator.xul"
  69 #else
  70 BROWSER_CHROME_URL = "chrome://browser/content/browser.xul"
  71 #endif
  72
  73 # Max time in seconds to wait for server startup before tests will fail -- if
  74 # this seems big, it's mostly for debug machines where cold startup
  75 # (particularly after a build) takes forever.
  76 SERVER_STARTUP_TIMEOUT = 45
  77
  78 INFINITY = 1.0e3000
  79
  80 oldcwd = os.getcwd()
  81 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
  82 os.chdir(SCRIPT_DIRECTORY)
  83
  84 PROFILE_DIRECTORY = os.path.abspath("./mochitesttestingprofile")
  85
  86 LEAK_REPORT_FILE = PROFILE_DIRECTORY + "/" + "leaks-report.log"
  87
  88 log = logging.getLogger()
  89
  90
  91 #######################
  92 # COMMANDLINE OPTIONS #
  93 #######################
  94
  95 class MochitestOptions(optparse.OptionParser):
  96   """Parses Mochitest commandline options."""
  97   def __init__(self, **kwargs):
  98     optparse.OptionParser.__init__(self, **kwargs)
  99     defaults = {}
 100
 101     self.add_option("--close-when-done",
 102                     action = "store_true", dest = "closeWhenDone",
 103                     help = "close the application when tests are done running")
 104     defaults["closeWhenDone"] = False
 105
 106     self.add_option("--appname",
 107                     action = "store", type = "string", dest = "app",
 108                     help = "absolute path to application, overriding default")
 109     defaults["app"] = automation.DEFAULT_APP
 110
 111     self.add_option("--log-file",
 112                     action = "store", type = "string",
 113                     dest = "logFile", metavar = "FILE",
 114                     help = "file to which logging occurs")
 115     defaults["logFile"] = ""
 116
 117     self.add_option("--autorun",
 118                     action = "store_true", dest = "autorun",
 119                     help = "start running tests when the application starts")
 120     defaults["autorun"] = False
 121
 122     LOG_LEVELS = ("DEBUG", "INFO", "WARNING", "ERROR", "FATAL")
 123     LEVEL_STRING = ", ".join(LOG_LEVELS)
 124
 125     self.add_option("--console-level",
 126                     action = "store", type = "choice", dest = "consoleLevel",
 127                     choices = LOG_LEVELS, metavar = "LEVEL",
 128                     help = "one of %s to determine the level of console "
 129                            "logging" % LEVEL_STRING)
 130     defaults["consoleLevel"] = None
 131
 132     self.add_option("--file-level",
 133                     action = "store", type = "choice", dest = "fileLevel",
 134                     choices = LOG_LEVELS, metavar = "LEVEL",
 135                     help = "one of %s to determine the level of file "
 136                            "logging if a file has been specified, defaulting "
 137                            "to INFO" % LEVEL_STRING)
 138     defaults["fileLevel"] = "INFO"
 139
 140     self.add_option("--chrome",
 141                     action = "store_true", dest = "chrome",
 142                     help = "run chrome Mochitests")
 143     defaults["chrome"] = False
 144
 145     self.add_option("--test-path",
 146                     action = "store", type = "string", dest = "testPath",
 147                     help = "start in the given directory's tests")
 148     defaults["testPath"] = ""
 149
 150     self.add_option("--browser-chrome",
 151                     action = "store_true", dest = "browserChrome",
 152                     help = "run browser chrome Mochitests")
 153     defaults["browserChrome"] = False
 154
 155     self.add_option("--a11y",
 156                     action = "store_true", dest = "a11y",
 157                     help = "run accessibility Mochitests");
 158
 159     self.add_option("--setenv",
 160                     action = "append", type = "string",
 161                     dest = "environment", metavar = "NAME=VALUE",
 162                     help = "sets the given variable in the application's "
 163                            "environment")
 164     defaults["environment"] = []
 165
 166     self.add_option("--browser-arg",
 167                     action = "append", type = "string",
 168                     dest = "browserArgs", metavar = "ARG",
 169                     help = "provides an argument to the test application")
 170     defaults["browserArgs"] = []
 171
 172     self.add_option("--leak-threshold",
 173                     action = "store", type = "int",
 174                     dest = "leakThreshold", metavar = "THRESHOLD",
 175                     help = "fail if the number of bytes leaked through "
 176                            "refcounted objects (or bytes in classes with "
 177                            "MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) is greater "
 178                            "than the given number")
 179     defaults["leakThreshold"] = INFINITY
 180
 181     self.add_option("--fatal-assertions",
 182                     action = "store_true", dest = "fatalAssertions",
 183                     help = "abort testing whenever an assertion is hit "
 184                            "(requires a debug build to be effective)")
 185     defaults["fatalAssertions"] = False
 186
 187     # -h, --help are automatically handled by OptionParser
 188
 189     self.set_defaults(**defaults)
 190
 191     usage = """\
 192 Usage instructions for runtests.py.
 193 All arguments are optional.
 194 If --chrome is specified, chrome tests will be run instead of web content tests.
 195 If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests.
 196 See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logging levels."""
 197     self.set_usage(usage)
 198
 199
 200
 201 #######################
 202 # HTTP SERVER SUPPORT #
 203 #######################
 204
 205 class MochitestServer:
 206   "Web server used to serve Mochitests, for closer fidelity to the real web."
 207
 208   def __init__(self, options):
 209     self._closeWhenDone = options.closeWhenDone
 210
 211   def start(self):
 212     "Run the Mochitest server, returning the process ID of the server."
 213
 214     env = dict(os.environ)
 215     if automation.UNIXISH:
 216       env["LD_LIBRARY_PATH"] = automation.DIST_BIN
 217       env["MOZILLA_FIVE_HOME"] = automation.DIST_BIN
 218       env["XPCOM_DEBUG_BREAK"] = "warn"
 219
 220     args = ["-v", "170",
 221             "-f", "./" + "httpd.js",
 222             "-f", "./" + "server.js"]
 223
 224     xpcshell = automation.DIST_BIN + "/" + "xpcshell";
 225     self._process = automation.Process(xpcshell, args, env = env)
 226     pid = self._process.pid
 227     if pid < 0:
 228       print "Error starting server."
 229       sys.exit(2)
 230     log.info("Server pid: %d", pid)
 231
 232
 233   def ensureReady(self, timeout):
 234     assert timeout >= 0
 235
 236     aliveFile = PROFILE_DIRECTORY + "/" + "server_alive.txt"
 237     i = 0
 238     while i < timeout:
 239       if os.path.exists(aliveFile):
 240         break
 241       time.sleep(1)
 242       i += 1
 243     else:
 244       print "Timed out while waiting for server startup."
 245       self.stop()
 246       sys.exit(1)
 247
 248   def stop(self):
 249     try:
 250       c = urllib2.urlopen(SERVER_SHUTDOWN_URL)
 251       c.read()
 252       c.close()
 253       self._process.wait()
 254     except:
 255       self._process.kill()
 256
 257
 258 #################
 259 # MAIN FUNCTION #
 260 #################
 261
 262 def main():
 263   parser = MochitestOptions()
 264   options, args = parser.parse_args()
 265
 266   # If the leak threshold wasn't explicitly set, we override the default of
 267   # infinity when the set of tests we're running are known to leak only a
 268   # particular amount.  If for some reason you don't want a new leak threshold
 269   # enforced, just pass an explicit --leak-threshold=N to prevent the override.
 270   maybeForceLeakThreshold(options)
 271
 272   if not os.path.exists(options.app):
 273     msg = """\
 274 Error: Path %(app)s doesn't exist.
 275 Are you executing $objdir/_tests/testing/mochitest/runtests.py?"""
 276     print msg % {"app": options.app}
 277     sys.exit(1)
 278
 279   # browser environment
 280   browserEnv = dict(os.environ)
 281
 282   # These variables are necessary for correct application startup; change
 283   # via the commandline at your own risk.
 284   browserEnv["NO_EM_RESTART"] = "1"
 285   browserEnv["XPCOM_DEBUG_BREAK"] = "warn"
 286   if automation.UNIXISH:
 287     browserEnv["LD_LIBRARY_PATH"] = automation.DIST_BIN
 288     browserEnv["MOZILLA_FIVE_HOME"] = automation.DIST_BIN
 289     browserEnv["GNOME_DISABLE_CRASH_DIALOG"] = "1"
 290
 291   for v in options.environment:
 292     ix = v.find("=")
 293     if ix <= 0:
 294       print "Error: syntax error in --setenv=" + v
 295       sys.exit(1)
 296     browserEnv[v[:ix]] = v[ix + 1:]
 297
 298   automation.initializeProfile(PROFILE_DIRECTORY)
 299   manifest = addChromeToProfile(options)
 300   server = MochitestServer(options)
 301   server.start()
 302
 303   # If we're lucky, the server has fully started by now, and all paths are
 304   # ready, etc.  However, xpcshell cold start times suck, at least for debug
 305   # builds.  We'll try to connect to the server for awhile, and if we fail,
 306   # we'll try to kill the server and exit with an error.
 307   server.ensureReady(SERVER_STARTUP_TIMEOUT)
 308
 309
 310   # URL parameters to test URL:
 311   #
 312   # autorun -- kick off tests automatically
 313   # closeWhenDone -- runs quit.js after tests
 314   # logFile -- logs test run to an absolute path
 315   #
 316
 317   # consoleLevel, fileLevel: set the logging level of the console and
 318   # file logs, if activated.
 319   # <http://mochikit.com/doc/html/MochiKit/Logging.html>
 320
 321   testURL = TESTS_URL + options.testPath
 322   urlOpts = []
 323   if options.chrome:
 324     testURL = CHROMETESTS_URL
 325     if options.testPath:
 326       urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
 327   elif options.a11y:
 328     testURL = A11YTESTS_URL
 329     if options.testPath:
 330       urlOpts.append("testPath=" + encodeURIComponent(options.testPath))
 331   elif options.browserChrome:
 332     testURL = "about:blank"
 333
 334   # allow relative paths for logFile
 335   if options.logFile:
 336     options.logFile = os.path.normpath(os.path.join(oldcwd, options.logFile))
 337   if options.browserChrome:
 338     makeTestConfig(options)
 339   else:
 340     if options.autorun:
 341       urlOpts.append("autorun=1")
 342     if options.closeWhenDone:
 343       urlOpts.append("closeWhenDone=1")
 344     if options.logFile:
 345       urlOpts.append("logFile=" + encodeURIComponent(options.logFile))
 346       urlOpts.append("fileLevel=" + encodeURIComponent(options.fileLevel))
 347     if options.consoleLevel:
 348       urlOpts.append("consoleLevel=" + encodeURIComponent(options.consoleLevel))
 349     if len(urlOpts) > 0:
 350       testURL += "?" + "&".join(urlOpts)
 351
 352   browserEnv["XPCOM_MEM_BLOAT_LOG"] = LEAK_REPORT_FILE
 353
 354   if options.fatalAssertions:
 355     browserEnv["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
 356
 357   start = automation.runApp(testURL, browserEnv, options.app, PROFILE_DIRECTORY,
 358                             options.browserArgs)
 359
 360   # Server's no longer needed, and perhaps more importantly, anything it might
 361   # spew to console shouldn't disrupt the leak information table we print next.
 362   server.stop()
 363
 364   if not os.path.exists(LEAK_REPORT_FILE):
 365     log.info("WARNING refcount logging is off, so leaks can't be detected!")
 366   else:
 367     leaks = open(LEAK_REPORT_FILE, "r")
 368     for line in leaks:
 369       log.info(line.rstrip())
 370     leaks.close()
 371
 372     threshold = options.leakThreshold
 373     leaks = open(LEAK_REPORT_FILE, "r")
 374     #                  Per-Inst  Leaked      Total  Rem ...
 375     #   0 TOTAL              17     192  419115886    2 ...
 376     # 833 nsTimerImpl        60     120      24726    2 ...
 377     lineRe = re.compile(r"^\s*\d+\s+(?P<name>\S+)\s+"
 378                         r"(?P<size>-?\d+)\s+(?P<bytesLeaked>-?\d+)\s+"
 379                         r"\d+\s+(?P<numLeaked>-?\d+)")
 380     seenTotal = False
 381     prefix = "TEST-PASS"
 382     for line in leaks:
 383       matches = lineRe.match(line)
 384       if not matches:
 385         continue
 386       name = matches.group("name")
 387       size = int(matches.group("size"))
 388       bytesLeaked = int(matches.group("bytesLeaked"))
 389       numLeaked = int(matches.group("numLeaked"))
 390       if size < 0 or bytesLeaked < 0 or numLeaked < 0:
 391         log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | negative leaks caught!")
 392       if "TOTAL" == name:
 393         seenTotal = True
 394         # Check for leaks.
 395         if bytesLeaked < 0 or bytesLeaked > threshold:
 396           prefix = "TEST-UNEXPECTED-FAIL"
 397           leakLog = "TEST-UNEXPECTED-FAIL | runtests-leaks | leaked" \
 398                     " %d bytes during test execution" % bytesLeaked
 399         elif bytesLeaked > 0:
 400           leakLog = "TEST-PASS | runtests-leaks | WARNING leaked" \
 401                     " %d bytes during test execution" % bytesLeaked
 402         else:
 403           leakLog = "TEST-PASS | runtests-leaks | no leaks detected!"
 404         # Remind the threshold if it is not 0, which is the goal.
 405         if threshold != 0:
 406           leakLog += (threshold == INFINITY) \
 407                      and (" (no threshold set)") \
 408                      or  (" (threshold set at %d bytes)" % threshold)
 409         # Log the information.
 410         log.info(leakLog)
 411       else:
 412         if numLeaked != 0:
 413           if abs(numLeaked) > 1:
 414             instance = "instances"
 415             rest = " each (%s bytes total)" % matches.group("bytesLeaked")
 416           else:
 417             instance = "instance"
 418             rest = ""
 419           log.info("%(prefix)s | runtests-leaks | leaked %(numLeaked)d %(instance)s of %(name)s "
 420                    "with size %(size)s bytes%(rest)s" %
 421                    { "prefix": prefix,
 422                      "numLeaked": numLeaked,
 423                      "instance": instance,
 424                      "name": name,
 425                      "size": matches.group("size"),
 426                      "rest": rest })
 427     if not seenTotal:
 428       log.info("TEST-UNEXPECTED-FAIL | runtests-leaks | missing output line for total leaks!")
 429     leaks.close()
 430
 431
 432   # print test run times
 433   finish = datetime.now()
 434   log.info(" started: %s", str(start))
 435   log.info("finished: %s", str(finish))
 436
 437   # delete the profile and manifest
 438   os.remove(manifest)
 439
 440   # hanging due to non-halting threads is no fun; assume we hit the errors we
 441   # were going to hit already and exit with a success code
 442   sys.exit(0)
 443
 444
 445
 446 #######################
 447 # CONFIGURATION SETUP #
 448 #######################
 449
 450 def maybeForceLeakThreshold(options):
 451   """
 452   Modifies an unset leak threshold if it is known that a particular leak
 453   threshold can be successfully forced for the particular Mochitest type
 454   and platform in use.
 455   """
 456   if options.leakThreshold == INFINITY:
 457     if options.chrome:
 458       # We don't leak running the --chrome tests.
 459       options.leakThreshold = 0
 460     elif options.browserChrome:
 461       # We still leak a nondeterministic amount running browser-chrome tests.
 462       # But we are close to 0 (bug), so start to prevent/detect regressions. (Bug 460548)
 463       options.leakThreshold = 75000
 464     elif options.a11y:
 465       # We don't leak running the --a11y tests.
 466       options.leakThreshold = 0
 467     else:
 468       # Normal Mochitests: no leaks.
 469       options.leakThreshold = 0
 470
 471 def makeTestConfig(options):
 472   "Creates a test configuration file for customizing test execution."
 473   def boolString(b):
 474     if b:
 475       return "true"
 476     return "false"
 477
 478   logFile = options.logFile.replace("\\", "\\\\")
 479   testPath = options.testPath.replace("\\", "\\\\")
 480   content = """\
 481 ({
 482   autoRun: %(autorun)s,
 483   closeWhenDone: %(closeWhenDone)s,
 484   logPath: "%(logPath)s",
 485   testPath: "%(testPath)s"
 486 })""" % {"autorun": boolString(options.autorun),
 487          "closeWhenDone": boolString(options.closeWhenDone),
 488          "logPath": logFile,
 489          "testPath": testPath}
 490
 491   config = open(PROFILE_DIRECTORY + "/" + "testConfig.js", "w")
 492   config.write(content)
 493   config.close()
 494
 495
 496 def addChromeToProfile(options):
 497   "Adds MochiKit chrome tests to the profile."
 498
 499   chromedir = PROFILE_DIRECTORY + "/" + "chrome"
 500   os.mkdir(chromedir)
 501
 502   chrome = []
 503
 504   part = """
 505 @namespace url("http://www.mozilla.org/keymaster/gatekeeper/there.is.only.xul"); /* set default namespace to XUL */
 506 toolbar,
 507 toolbarpalette {
 508   background-color: rgb(235, 235, 235) !important;
 509 }
 510 toolbar#nav-bar {
 511   background-image: none !important;
 512 }
 513 """
 514   chrome.append(part)
 515
 516
 517
 518   # write userChrome.css
 519   chromeFile = open(PROFILE_DIRECTORY + "/" + "userChrome.css", "a")
 520   chromeFile.write("".join(chrome))
 521   chromeFile.close()
 522
 523
 524   # register our chrome dir
 525   chrometestDir = os.path.abspath(".") + "/"
 526   if automation.IS_WIN32:
 527     chrometestDir = "file:///" + chrometestDir.replace("\\", "/")
 528
 529
 530   (path, leaf) = os.path.split(options.app)
 531   manifest = path + "/" + "chrome/mochikit.manifest"
 532   manifestFile = open(manifest, "w")
 533   manifestFile.write("content mochikit " + chrometestDir + " contentaccessible=yes\n")
 534   if options.browserChrome:
 535     overlayLine = "overlay " + BROWSER_CHROME_URL + " " \
 536                   "chrome://mochikit/content/browser-test-overlay.xul\n"
 537     manifestFile.write(overlayLine)
 538   manifestFile.close()
 539
 540   return manifest
 541
 542 #########
 543 # DO IT #
 544 #########
 545
 546 if __name__ == "__main__":
 547   main()