tools/heapcheck/chrome_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 ''' Runs various chrome tests through heapcheck_test.py.
   7
   8 Most of this code is copied from ../valgrind/chrome_tests.py.
   9 TODO(glider): put common functions to a standalone module.
  10 '''
  11
  12 import glob
  13 import logging
  14 import optparse
  15 import os
  16 import stat
  17 import sys
  18
  19 import logging_utils
  20 import path_utils
  21
  22 import common
  23 import heapcheck_test
  24
  25 class TestNotFound(Exception): pass
  26
  27 def Dir2IsNewer(dir1, dir2):
  28   if dir2 is None or not os.path.isdir(dir2):
  29     return False
  30   if dir1 is None or not os.path.isdir(dir1):
  31     return True
  32   return os.stat(dir2)[stat.ST_MTIME] > os.stat(dir1)[stat.ST_MTIME]
  33
  34 def FindNewestDir(dirs):
  35   newest_dir = None
  36   for dir in dirs:
  37     if Dir2IsNewer(newest_dir, dir):
  38       newest_dir = dir
  39   return newest_dir
  40
  41 def File2IsNewer(file1, file2):
  42   if file2 is None or not os.path.isfile(file2):
  43     return False
  44   if file1 is None or not os.path.isfile(file1):
  45     return True
  46   return os.stat(file2)[stat.ST_MTIME] > os.stat(file1)[stat.ST_MTIME]
  47
  48 def FindDirContainingNewestFile(dirs, file):
  49   """Searches for the directory containing the newest copy of |file|.
  50
  51   Args:
  52     dirs: A list of paths to the directories to search among.
  53     file: A string containing the file name to search.
  54
  55   Returns:
  56     The string representing the the directory containing the newest copy of
  57     |file|.
  58
  59   Raises:
  60     IOError: |file| was not found.
  61   """
  62   newest_dir = None
  63   newest_file = None
  64   for dir in dirs:
  65     the_file = os.path.join(dir, file)
  66     if File2IsNewer(newest_file, the_file):
  67       newest_dir = dir
  68       newest_file = the_file
  69   if newest_dir is None:
  70     raise IOError("cannot find file %s anywhere, have you built it?" % file)
  71   return newest_dir
  72
  73 class ChromeTests(object):
  74   '''This class is derived from the chrome_tests.py file in ../purify/.
  75   '''
  76
  77   def __init__(self, options, args, test):
  78     # The known list of tests.
  79     # Recognise the original abbreviations as well as full executable names.
  80     self._test_list = {
  81       "ash": self.TestAsh,              "ash_unittests": self.TestAsh,
  82       "aura": self.TestAura,            "aura_unittests": self.TestAura,
  83       "base": self.TestBase,            "base_unittests": self.TestBase,
  84       "browser": self.TestBrowser,      "browser_tests": self.TestBrowser,
  85       "chromeos": self.TestChromeOS,    "chromeos_unittests": self.TestChromeOS,
  86       "compositor": self.TestCompositor,
  87       "compositor_unittests": self.TestCompositor,
  88       "content": self.TestContent,      "content_unittests": self.TestContent,
  89       "content_browsertests": self.TestContentBrowser,
  90       "courgette": self.TestCourgette,
  91       "courgette_unittests": self.TestCourgette,
  92       "crypto": self.TestCrypto,        "crypto_unittests": self.TestCrypto,
  93       "device": self.TestDevice,        "device_unittests": self.TestDevice,
  94       "googleurl": self.TestGURL,       "googleurl_unittests": self.TestGURL,
  95       "url": self.TestURL,              "url_unittests": self.TestURL,
  96       "ipc": self.TestIpc,              "ipc_tests": self.TestIpc,
  97       "layout": self.TestLayout,        "layout_tests": self.TestLayout,
  98       "media": self.TestMedia,          "media_unittests": self.TestMedia,
  99       "net": self.TestNet,              "net_unittests": self.TestNet,
 100       "printing": self.TestPrinting,    "printing_unittests": self.TestPrinting,
 101       "remoting": self.TestRemoting,    "remoting_unittests": self.TestRemoting,
 102       "sql": self.TestSql,              "sql_unittests": self.TestSql,
 103       "startup": self.TestStartup,      "startup_tests": self.TestStartup,
 104       "sync": self.TestSync,            "sync_unit_tests": self.TestSync,
 105       "ui_unit": self.TestUIUnit,       "ui_unittests": self.TestUIUnit,
 106       "unit": self.TestUnit,            "unit_tests": self.TestUnit,
 107       "views": self.TestViews,          "views_unittests": self.TestViews,
 108     }
 109
 110     if test not in self._test_list:
 111       raise TestNotFound("Unknown test: %s" % test)
 112
 113     self._options = options
 114     self._args = args
 115     self._test = test
 116
 117     script_dir = path_utils.ScriptDir()
 118
 119     # Compute the top of the tree (the "source dir") from the script dir (where
 120     # this script lives).  We assume that the script dir is in tools/heapcheck/
 121     # relative to the top of the tree.
 122     self._source_dir = os.path.dirname(os.path.dirname(script_dir))
 123
 124     # Since this path is used for string matching, make sure it's always
 125     # an absolute Unix-style path.
 126     self._source_dir = os.path.abspath(self._source_dir).replace('\\', '/')
 127
 128     heapcheck_test_script = os.path.join(script_dir, "heapcheck_test.py")
 129     self._command_preamble = [heapcheck_test_script]
 130
 131   def _DefaultCommand(self, module, exe=None, heapcheck_test_args=None):
 132     '''Generates the default command array that most tests will use.
 133
 134     Args:
 135       module: The module name (corresponds to the dir in src/ where the test
 136               data resides).
 137       exe: The executable name.
 138       heapcheck_test_args: additional arguments to append to the command line.
 139     Returns:
 140       A string with the command to run the test.
 141     '''
 142     if not self._options.build_dir:
 143       dirs = [
 144         os.path.join(self._source_dir, "xcodebuild", "Debug"),
 145         os.path.join(self._source_dir, "out", "Debug"),
 146       ]
 147       if exe:
 148         self._options.build_dir = FindDirContainingNewestFile(dirs, exe)
 149       else:
 150         self._options.build_dir = FindNewestDir(dirs)
 151
 152     cmd = list(self._command_preamble)
 153
 154     if heapcheck_test_args != None:
 155       for arg in heapcheck_test_args:
 156         cmd.append(arg)
 157     if exe:
 158       cmd.append(os.path.join(self._options.build_dir, exe))
 159       # Heapcheck runs tests slowly, so slow tests hurt more; show elapased time
 160       # so we can find the slowpokes.
 161       cmd.append("--gtest_print_time")
 162     if self._options.gtest_repeat:
 163       cmd.append("--gtest_repeat=%s" % self._options.gtest_repeat)
 164     return cmd
 165
 166   def Suppressions(self):
 167     '''Builds the list of available suppressions files.'''
 168     ret = []
 169     directory = path_utils.ScriptDir()
 170     suppression_file = os.path.join(directory, "suppressions.txt")
 171     if os.path.exists(suppression_file):
 172       ret.append(suppression_file)
 173     suppression_file = os.path.join(directory, "suppressions_linux.txt")
 174     if os.path.exists(suppression_file):
 175       ret.append(suppression_file)
 176     return ret
 177
 178   def Run(self):
 179     '''Runs the test specified by command-line argument --test.'''
 180     logging.info("running test %s" % (self._test))
 181     return self._test_list[self._test]()
 182
 183   def _ReadGtestFilterFile(self, name, cmd):
 184     '''Reads files which contain lists of tests to filter out with
 185     --gtest_filter and appends the command-line option to |cmd|.
 186
 187     Args:
 188       name: the test executable name.
 189       cmd: the test running command line to be modified.
 190     '''
 191     filters = []
 192     directory = path_utils.ScriptDir()
 193     gtest_filter_files = [
 194         os.path.join(directory, name + ".gtest-heapcheck.txt"),
 195         # TODO(glider): Linux vs. CrOS?
 196     ]
 197     logging.info("Reading gtest exclude filter files:")
 198     for filename in gtest_filter_files:
 199       # strip the leading absolute path (may be very long on the bot)
 200       # and the following / or \.
 201       readable_filename = filename.replace(self._source_dir, "")[1:]
 202       if not os.path.exists(filename):
 203         logging.info("  \"%s\" - not found" % readable_filename)
 204         continue
 205       logging.info("  \"%s\" - OK" % readable_filename)
 206       f = open(filename, 'r')
 207       for line in f.readlines():
 208         if line.startswith("#") or line.startswith("//") or line.isspace():
 209           continue
 210         line = line.rstrip()
 211         filters.append(line)
 212     gtest_filter = self._options.gtest_filter
 213     if len(filters):
 214       if gtest_filter:
 215         gtest_filter += ":"
 216         if gtest_filter.find("-") < 0:
 217           gtest_filter += "-"
 218       else:
 219         gtest_filter = "-"
 220       gtest_filter += ":".join(filters)
 221     if gtest_filter:
 222       cmd.append("--gtest_filter=%s" % gtest_filter)
 223
 224   def SimpleTest(self, module, name, heapcheck_test_args=None, cmd_args=None):
 225     '''Builds the command line and runs the specified test.
 226
 227     Args:
 228       module: The module name (corresponds to the dir in src/ where the test
 229               data resides).
 230       name: The executable name.
 231       heapcheck_test_args: Additional command line args for heap checker.
 232       cmd_args: Additional command line args for the test.
 233     '''
 234     cmd = self._DefaultCommand(module, name, heapcheck_test_args)
 235     supp = self.Suppressions()
 236     self._ReadGtestFilterFile(name, cmd)
 237     if cmd_args:
 238       cmd.extend(["--"])
 239       cmd.extend(cmd_args)
 240
 241     # Sets LD_LIBRARY_PATH to the build folder so external libraries can be
 242     # loaded.
 243     if (os.getenv("LD_LIBRARY_PATH")):
 244       os.putenv("LD_LIBRARY_PATH", "%s:%s" % (os.getenv("LD_LIBRARY_PATH"),
 245                                               self._options.build_dir))
 246     else:
 247       os.putenv("LD_LIBRARY_PATH", self._options.build_dir)
 248     return heapcheck_test.RunTool(cmd, supp, module)
 249
 250   def TestAsh(self):
 251     return self.SimpleTest("ash", "ash_unittests")
 252
 253   def TestAura(self):
 254     return self.SimpleTest("aura", "aura_unittests")
 255
 256   def TestBase(self):
 257     return self.SimpleTest("base", "base_unittests")
 258
 259   def TestBrowser(self):
 260     return self.SimpleTest("chrome", "browser_tests")
 261
 262   def TestChromeOS(self):
 263     return self.SimpleTest("chromeos", "chromeos_unittests")
 264
 265   def TestCompositor(self):
 266     return self.SimpleTest("compositor", "compositor_unittests")
 267
 268   def TestContent(self):
 269     return self.SimpleTest("content", "content_unittests")
 270
 271   def TestContentBrowser(self):
 272     return self.SimpleTest("content", "content_browsertests")
 273
 274   def TestCourgette(self):
 275     return self.SimpleTest("courgette", "courgette_unittests")
 276
 277   def TestCrypto(self):
 278     return self.SimpleTest("crypto", "crypto_unittests")
 279
 280   def TestDevice(self):
 281     return self.SimpleTest("device", "device_unittests")
 282
 283   def TestGURL(self):
 284     return self.SimpleTest("chrome", "googleurl_unittests")
 285
 286   def TestURL(self):
 287     return self.SimpleTest("chrome", "url_unittests")
 288
 289   def TestIpc(self):
 290     return self.SimpleTest("ipc", "ipc_tests")
 291
 292   def TestMedia(self):
 293     return self.SimpleTest("chrome", "media_unittests")
 294
 295   def TestNet(self):
 296     return self.SimpleTest("net", "net_unittests")
 297
 298   def TestPrinting(self):
 299     return self.SimpleTest("chrome", "printing_unittests")
 300
 301   def TestRemoting(self):
 302     return self.SimpleTest("chrome", "remoting_unittests")
 303
 304   def TestSync(self):
 305     return self.SimpleTest("chrome", "sync_unit_tests")
 306
 307   def TestStartup(self):
 308     # We don't need the performance results, we're just looking for pointer
 309     # errors, so set number of iterations down to the minimum.
 310     os.putenv("STARTUP_TESTS_NUMCYCLES", "1")
 311     logging.info("export STARTUP_TESTS_NUMCYCLES=1");
 312     return self.SimpleTest("chrome", "startup_tests")
 313
 314   def TestUIUnit(self):
 315     return self.SimpleTest("chrome", "ui_unittests")
 316
 317   def TestUnit(self):
 318     return self.SimpleTest("chrome", "unit_tests")
 319
 320   def TestSql(self):
 321     return self.SimpleTest("chrome", "sql_unittests")
 322
 323   def TestViews(self):
 324     return self.SimpleTest("views", "views_unittests")
 325
 326   def TestLayoutChunk(self, chunk_num, chunk_size):
 327     '''Runs tests [chunk_num*chunk_size .. (chunk_num+1)*chunk_size).
 328
 329     Wrap around to beginning of list at end. If chunk_size is zero, run all
 330     tests in the list once. If a text file is given as argument, it is used as
 331     the list of tests.
 332     '''
 333     # Build the ginormous commandline in 'cmd'.
 334     # It's going to be roughly
 335     #  python heapcheck_test.py ... python run_webkit_tests.py ...
 336     # but we'll use the --indirect flag to heapcheck_test.py
 337     # to avoid heapchecking python.
 338     # Start by building the heapcheck_test.py commandline.
 339     cmd = self._DefaultCommand("webkit")
 340
 341     # Now build script_cmd, the run_webkits_tests.py commandline
 342     # Store each chunk in its own directory so that we can find the data later
 343     chunk_dir = os.path.join("layout", "chunk_%05d" % chunk_num)
 344     out_dir = os.path.join(path_utils.ScriptDir(), "latest")
 345     out_dir = os.path.join(out_dir, chunk_dir)
 346     if os.path.exists(out_dir):
 347       old_files = glob.glob(os.path.join(out_dir, "*.txt"))
 348       for f in old_files:
 349         os.remove(f)
 350     else:
 351       os.makedirs(out_dir)
 352
 353     script = os.path.join(self._source_dir, "webkit", "tools", "layout_tests",
 354                           "run_webkit_tests.py")
 355     script_cmd = ["python", script, "--run-singly", "-v",
 356                   "--noshow-results", "--time-out-ms=200000",
 357                   "--nocheck-sys-deps"]
 358
 359     # Pass build mode to run_webkit_tests.py.  We aren't passed it directly,
 360     # so parse it out of build_dir.  run_webkit_tests.py can only handle
 361     # the two values "Release" and "Debug".
 362     # TODO(Hercules): unify how all our scripts pass around build mode
 363     # (--mode / --target / --build_dir / --debug)
 364     if self._options.build_dir.endswith("Debug"):
 365       script_cmd.append("--debug");
 366     if (chunk_size > 0):
 367       script_cmd.append("--run-chunk=%d:%d" % (chunk_num, chunk_size))
 368     if len(self._args):
 369       # if the arg is a txt file, then treat it as a list of tests
 370       if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt":
 371         script_cmd.append("--test-list=%s" % self._args[0])
 372       else:
 373         script_cmd.extend(self._args)
 374     self._ReadGtestFilterFile("layout", script_cmd)
 375
 376     # Now run script_cmd with the wrapper in cmd
 377     cmd.extend(["--"])
 378     cmd.extend(script_cmd)
 379     supp = self.Suppressions()
 380     return heapcheck_test.RunTool(cmd, supp, "layout")
 381
 382   def TestLayout(self):
 383     '''Runs the layout tests.'''
 384     # A "chunk file" is maintained in the local directory so that each test
 385     # runs a slice of the layout tests of size chunk_size that increments with
 386     # each run.  Since tests can be added and removed from the layout tests at
 387     # any time, this is not going to give exact coverage, but it will allow us
 388     # to continuously run small slices of the layout tests under purify rather
 389     # than having to run all of them in one shot.
 390     chunk_size = self._options.num_tests
 391     if (chunk_size == 0):
 392       return self.TestLayoutChunk(0, 0)
 393     chunk_num = 0
 394     chunk_file = os.path.join("heapcheck_layout_chunk.txt")
 395
 396     logging.info("Reading state from " + chunk_file)
 397     try:
 398       f = open(chunk_file)
 399       if f:
 400         str = f.read()
 401         if len(str):
 402           chunk_num = int(str)
 403         # This should be enough so that we have a couple of complete runs
 404         # of test data stored in the archive (although note that when we loop
 405         # that we almost guaranteed won't be at the end of the test list)
 406         if chunk_num > 10000:
 407           chunk_num = 0
 408         f.close()
 409     except IOError, (errno, strerror):
 410       logging.error("error reading from file %s (%d, %s)" % (chunk_file,
 411                     errno, strerror))
 412     ret = self.TestLayoutChunk(chunk_num, chunk_size)
 413
 414     # Wait until after the test runs to completion to write out the new chunk
 415     # number.  This way, if the bot is killed, we'll start running again from
 416     # the current chunk rather than skipping it.
 417     logging.info("Saving state to " + chunk_file)
 418     try:
 419       f = open(chunk_file, "w")
 420       chunk_num += 1
 421       f.write("%d" % chunk_num)
 422       f.close()
 423     except IOError, (errno, strerror):
 424       logging.error("error writing to file %s (%d, %s)" % (chunk_file, errno,
 425                     strerror))
 426
 427     # Since we're running small chunks of the layout tests, it's important to
 428     # mark the ones that have errors in them.  These won't be visible in the
 429     # summary list for long, but will be useful for someone reviewing this bot.
 430     return ret
 431
 432
 433 def main():
 434   if not sys.platform.startswith('linux'):
 435     logging.error("Heap checking works only on Linux at the moment.")
 436     return 1
 437   parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> "
 438                                  "[-t <test> ...]")
 439   parser.disable_interspersed_args()
 440   parser.add_option("-b", "--build_dir",
 441                     help="the location of the output of the compiler output")
 442   parser.add_option("-t", "--test", action="append",
 443                     help="which test to run")
 444   parser.add_option("", "--gtest_filter",
 445                     help="additional arguments to --gtest_filter")
 446   parser.add_option("", "--gtest_repeat",
 447                     help="argument for --gtest_repeat")
 448   parser.add_option("-v", "--verbose", action="store_true", default=False,
 449                     help="verbose output - enable debug log messages")
 450   # My machine can do about 120 layout tests/hour in release mode.
 451   # Let's do 30 minutes worth per run.
 452   # The CPU is mostly idle, so perhaps we can raise this when
 453   # we figure out how to run them more efficiently.
 454   parser.add_option("-n", "--num_tests", default=60, type="int",
 455                     help="for layout tests: # of subtests per run.  0 for all.")
 456
 457   options, args = parser.parse_args()
 458
 459   if options.verbose:
 460     logging_utils.config_root(logging.DEBUG)
 461   else:
 462     logging_utils.config_root()
 463
 464   if not options.test or not len(options.test):
 465     parser.error("--test not specified")
 466
 467   for t in options.test:
 468     tests = ChromeTests(options, args, t)
 469     ret = tests.Run()
 470     if ret:
 471       return ret
 472   return 0
 473
 474
 475 if __name__ == "__main__":
 476   sys.exit(main())