test/functional/test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Run regression test suite.
   6
   7 This module calls down into individual test cases via subprocess. It will
   8 forward all unrecognized arguments onto the individual test scripts.
   9
  10 Functional tests are disabled on Windows by default. Use --force to run them anyway.
  11
  12 For a description of arguments recognized by test scripts, see
  13 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
  14
  15 """
  16
  17 import argparse
  18 import configparser
  19 import datetime
  20 import os
  21 import time
  22 import shutil
  23 import signal
  24 import sys
  25 import subprocess
  26 import tempfile
  27 import re
  28 import logging
  29
  30 # Formatting. Default colors to empty strings.
  31 BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
  32 try:
  33     # Make sure python thinks it can write unicode to its stdout
  34     "\u2713".encode("utf_8").decode(sys.stdout.encoding)
  35     TICK = "✓ "
  36     CROSS = "✖ "
  37     CIRCLE = "○ "
  38 except UnicodeDecodeError:
  39     TICK = "P "
  40     CROSS = "x "
  41     CIRCLE = "o "
  42
  43 if os.name == 'posix':
  44     # primitive formatting on supported
  45     # terminal via ANSI escape sequences:
  46     BOLD = ('\033[0m', '\033[1m')
  47     BLUE = ('\033[0m', '\033[0;34m')
  48     RED = ('\033[0m', '\033[0;31m')
  49     GREY = ('\033[0m', '\033[1;30m')
  50
  51 TEST_EXIT_PASSED = 0
  52 TEST_EXIT_SKIPPED = 77
  53
  54 BASE_SCRIPTS= [
  55     # Scripts that are run by the travis build process.
  56     # Longest test should go first, to favor running tests in parallel
  57     'wallet-hd.py',
  58     'walletbackup.py',
  59     # vv Tests less than 5m vv
  60     'p2p-fullblocktest.py',
  61     'fundrawtransaction.py',
  62     'p2p-compactblocks.py',
  63     'segwit.py',
  64     # vv Tests less than 2m vv
  65     'wallet.py',
  66     'wallet-accounts.py',
  67     'p2p-segwit.py',
  68     'wallet-dump.py',
  69     'listtransactions.py',
  70     # vv Tests less than 60s vv
  71     'sendheaders.py',
  72     'zapwallettxes.py',
  73     'importmulti.py',
  74     'mempool_limit.py',
  75     'merkle_blocks.py',
  76     'receivedby.py',
  77     'abandonconflict.py',
  78     'bip68-112-113-p2p.py',
  79     'rawtransactions.py',
  80     'reindex.py',
  81     # vv Tests less than 30s vv
  82     'keypool-topup.py',
  83     'zmq_test.py',
  84     'bitcoin_cli.py',
  85     'mempool_resurrect_test.py',
  86     'txn_doublespend.py --mineblock',
  87     'txn_clone.py',
  88     'getchaintips.py',
  89     'rest.py',
  90     'mempool_spendcoinbase.py',
  91     'mempool_reorg.py',
  92     'mempool_persist.py',
  93     'multiwallet.py',
  94     'httpbasics.py',
  95     'multi_rpc.py',
  96     'proxy_test.py',
  97     'signrawtransactions.py',
  98     'disconnect_ban.py',
  99     'decodescript.py',
 100     'blockchain.py',
 101     'deprecated_rpc.py',
 102     'disablewallet.py',
 103     'net.py',
 104     'keypool.py',
 105     'p2p-mempool.py',
 106     'prioritise_transaction.py',
 107     'invalidblockrequest.py',
 108     'invalidtxrequest.py',
 109     'p2p-versionbits-warning.py',
 110     'preciousblock.py',
 111     'importprunedfunds.py',
 112     'signmessages.py',
 113     'nulldummy.py',
 114     'import-rescan.py',
 115     'mining.py',
 116     'bumpfee.py',
 117     'rpcnamedargs.py',
 118     'listsinceblock.py',
 119     'p2p-leaktests.py',
 120     'wallet-encryption.py',
 121     'bipdersig-p2p.py',
 122     'bip65-cltv-p2p.py',
 123     'uptime.py',
 124     'resendwallettransactions.py',
 125     'minchainwork.py',
 126 ]
 127
 128 EXTENDED_SCRIPTS = [
 129     # These tests are not run by the travis build process.
 130     # Longest test should go first, to favor running tests in parallel
 131     'pruning.py',
 132     # vv Tests less than 20m vv
 133     'smartfees.py',
 134     # vv Tests less than 5m vv
 135     'maxuploadtarget.py',
 136     'mempool_packages.py',
 137     'dbcrash.py',
 138     # vv Tests less than 2m vv
 139     'bip68-sequence.py',
 140     'getblocktemplate_longpoll.py',
 141     'p2p-timeouts.py',
 142     # vv Tests less than 60s vv
 143     'bip9-softforks.py',
 144     'p2p-feefilter.py',
 145     'rpcbind_test.py',
 146     # vv Tests less than 30s vv
 147     'assumevalid.py',
 148     'example_test.py',
 149     'txn_doublespend.py',
 150     'txn_clone.py --mineblock',
 151     'notifications.py',
 152     'invalidateblock.py',
 153     'p2p-acceptblock.py',
 154     'replace-by-fee.py',
 155 ]
 156
 157 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
 158 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
 159
 160 NON_SCRIPTS = [
 161     # These are python files that live in the functional tests directory, but are not test scripts.
 162     "combine_logs.py",
 163     "create_cache.py",
 164     "test_runner.py",
 165 ]
 166
 167 def main():
 168     # Parse arguments and pass through unrecognised args
 169     parser = argparse.ArgumentParser(add_help=False,
 170                                      usage='%(prog)s [test_runner.py options] [script options] [scripts]',
 171                                      description=__doc__,
 172                                      epilog='''
 173     Help text and arguments for individual test script:''',
 174                                      formatter_class=argparse.RawTextHelpFormatter)
 175     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
 176     parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.')
 177     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
 178     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
 179     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
 180     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
 181     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
 182     parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
 183     parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
 184     args, unknown_args = parser.parse_known_args()
 185
 186     # args to be passed on always start with two dashes; tests are the remaining unknown args
 187     tests = [arg for arg in unknown_args if arg[:2] != "--"]
 188     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 189
 190     # Read config generated by configure.
 191     config = configparser.ConfigParser()
 192     configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
 193     config.read_file(open(configfile))
 194
 195     passon_args.append("--configfile=%s" % configfile)
 196
 197     # Set up logging
 198     logging_level = logging.INFO if args.quiet else logging.DEBUG
 199     logging.basicConfig(format='%(message)s', level=logging_level)
 200
 201     # Create base test directory
 202     tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
 203     os.makedirs(tmpdir)
 204
 205     logging.debug("Temporary test directory at %s" % tmpdir)
 206
 207     enable_wallet = config["components"].getboolean("ENABLE_WALLET")
 208     enable_utils = config["components"].getboolean("ENABLE_UTILS")
 209     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 210
 211     if config["environment"]["EXEEXT"] == ".exe" and not args.force:
 212         # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
 213         # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
 214         print("Tests currently disabled on Windows by default. Use --force option to enable")
 215         sys.exit(0)
 216
 217     if not (enable_wallet and enable_utils and enable_bitcoind):
 218         print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled")
 219         print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
 220         sys.exit(0)
 221
 222     # Build list of tests
 223     if tests:
 224         # Individual tests have been specified. Run specified tests that exist
 225         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
 226         tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
 227         test_list = []
 228         for t in tests:
 229             if t in ALL_SCRIPTS:
 230                 test_list.append(t)
 231             else:
 232                 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
 233     else:
 234         # No individual tests have been specified.
 235         # Run all base tests, and optionally run extended tests.
 236         test_list = BASE_SCRIPTS
 237         if args.extended:
 238             # place the EXTENDED_SCRIPTS first since the three longest ones
 239             # are there and the list is shorter
 240             test_list = EXTENDED_SCRIPTS + test_list
 241
 242     # Remove the test cases that the user has explicitly asked to exclude.
 243     if args.exclude:
 244         tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
 245         for exclude_test in tests_excl:
 246             if exclude_test in test_list:
 247                 test_list.remove(exclude_test)
 248             else:
 249                 print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
 250
 251     if not test_list:
 252         print("No valid test scripts specified. Check that your test is in one "
 253               "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
 254         sys.exit(0)
 255
 256     if args.help:
 257         # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
 258         parser.print_help()
 259         subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
 260         sys.exit(0)
 261
 262     check_script_list(config["environment"]["SRCDIR"])
 263
 264     if not args.keepcache:
 265         shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
 266
 267     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args)
 268
 269 def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]):
 270     # Warn if bitcoind is already running (unix only)
 271     try:
 272         if subprocess.check_output(["pidof", "bitcoind"]) is not None:
 273             print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
 274     except (OSError, subprocess.SubprocessError):
 275         pass
 276
 277     # Warn if there is a cache directory
 278     cache_dir = "%s/test/cache" % build_dir
 279     if os.path.isdir(cache_dir):
 280         print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir))
 281
 282     #Set env vars
 283     if "BITCOIND" not in os.environ:
 284         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
 285         os.environ["BITCOINCLI"] = build_dir + '/src/bitcoin-cli' + exeext
 286
 287     tests_dir = src_dir + '/test/functional/'
 288
 289     flags = ["--srcdir={}/src".format(build_dir)] + args
 290     flags.append("--cachedir=%s" % cache_dir)
 291
 292     if enable_coverage:
 293         coverage = RPCCoverage()
 294         flags.append(coverage.flag)
 295         logging.debug("Initializing coverage directory at %s" % coverage.dir)
 296     else:
 297         coverage = None
 298
 299     if len(test_list) > 1 and jobs > 1:
 300         # Populate cache
 301         subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
 302
 303     #Run Tests
 304     job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags)
 305     time0 = time.time()
 306     test_results = []
 307
 308     max_len_name = len(max(test_list, key=len))
 309
 310     for _ in range(len(test_list)):
 311         test_result, stdout, stderr = job_queue.get_next()
 312         test_results.append(test_result)
 313
 314         if test_result.status == "Passed":
 315             logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 316         elif test_result.status == "Skipped":
 317             logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0]))
 318         else:
 319             print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 320             print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
 321             print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
 322
 323     print_results(test_results, max_len_name, (int(time.time() - time0)))
 324
 325     if coverage:
 326         coverage.report_rpc_coverage()
 327
 328         logging.debug("Cleaning up coverage data")
 329         coverage.cleanup()
 330
 331     # Clear up the temp directory if all subdirectories are gone
 332     if not os.listdir(tmpdir):
 333         os.rmdir(tmpdir)
 334
 335     all_passed = all(map(lambda test_result: test_result.was_successful, test_results))
 336
 337     sys.exit(not all_passed)
 338
 339 def print_results(test_results, max_len_name, runtime):
 340     results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS   ", "DURATION") + BOLD[0]
 341
 342     test_results.sort(key=lambda result: result.name.lower())
 343     all_passed = True
 344     time_sum = 0
 345
 346     for test_result in test_results:
 347         all_passed = all_passed and test_result.was_successful
 348         time_sum += test_result.time
 349         test_result.padding = max_len_name
 350         results += str(test_result)
 351
 352     status = TICK + "Passed" if all_passed else CROSS + "Failed"
 353     results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
 354     results += "Runtime: %s s\n" % (runtime)
 355     print(results)
 356
 357 class TestHandler:
 358     """
 359     Trigger the test scripts passed in via the list.
 360     """
 361
 362     def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None):
 363         assert(num_tests_parallel >= 1)
 364         self.num_jobs = num_tests_parallel
 365         self.tests_dir = tests_dir
 366         self.tmpdir = tmpdir
 367         self.test_list = test_list
 368         self.flags = flags
 369         self.num_running = 0
 370         # In case there is a graveyard of zombie bitcoinds, we can apply a
 371         # pseudorandom offset to hopefully jump over them.
 372         # (625 is PORT_RANGE/MAX_NODES)
 373         self.portseed_offset = int(time.time() * 1000) % 625
 374         self.jobs = []
 375
 376     def get_next(self):
 377         while self.num_running < self.num_jobs and self.test_list:
 378             # Add tests
 379             self.num_running += 1
 380             t = self.test_list.pop(0)
 381             portseed = len(self.test_list) + self.portseed_offset
 382             portseed_arg = ["--portseed={}".format(portseed)]
 383             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
 384             log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
 385             test_argv = t.split()
 386             tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)]
 387             self.jobs.append((t,
 388                               time.time(),
 389                               subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir,
 390                                                universal_newlines=True,
 391                                                stdout=log_stdout,
 392                                                stderr=log_stderr),
 393                               log_stdout,
 394                               log_stderr))
 395         if not self.jobs:
 396             raise IndexError('pop from empty list')
 397         while True:
 398             # Return first proc that finishes
 399             time.sleep(.5)
 400             for j in self.jobs:
 401                 (name, time0, proc, log_out, log_err) = j
 402                 if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60:
 403                     # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not
 404                     # providing useful output.
 405                     proc.send_signal(signal.SIGINT)
 406                 if proc.poll() is not None:
 407                     log_out.seek(0), log_err.seek(0)
 408                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
 409                     log_out.close(), log_err.close()
 410                     if proc.returncode == TEST_EXIT_PASSED and stderr == "":
 411                         status = "Passed"
 412                     elif proc.returncode == TEST_EXIT_SKIPPED:
 413                         status = "Skipped"
 414                     else:
 415                         status = "Failed"
 416                     self.num_running -= 1
 417                     self.jobs.remove(j)
 418
 419                     return TestResult(name, status, int(time.time() - time0)), stdout, stderr
 420             print('.', end='', flush=True)
 421
 422 class TestResult():
 423     def __init__(self, name, status, time):
 424         self.name = name
 425         self.status = status
 426         self.time = time
 427         self.padding = 0
 428
 429     def __repr__(self):
 430         if self.status == "Passed":
 431             color = BLUE
 432             glyph = TICK
 433         elif self.status == "Failed":
 434             color = RED
 435             glyph = CROSS
 436         elif self.status == "Skipped":
 437             color = GREY
 438             glyph = CIRCLE
 439
 440         return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
 441
 442     @property
 443     def was_successful(self):
 444         return self.status != "Failed"
 445
 446
 447 def check_script_list(src_dir):
 448     """Check scripts directory.
 449
 450     Check that there are no scripts in the functional tests directory which are
 451     not being run by pull-tester.py."""
 452     script_dir = src_dir + '/test/functional/'
 453     python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
 454     missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
 455     if len(missed_tests) != 0:
 456         print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
 457         if os.getenv('TRAVIS') == 'true':
 458             # On travis this warning is an error to prevent merging incomplete commits into master
 459             sys.exit(1)
 460
 461 class RPCCoverage(object):
 462     """
 463     Coverage reporting utilities for test_runner.
 464
 465     Coverage calculation works by having each test script subprocess write
 466     coverage files into a particular directory. These files contain the RPC
 467     commands invoked during testing, as well as a complete listing of RPC
 468     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 469
 470     After all tests complete, the commands run are combined and diff'd against
 471     the complete list to calculate uncovered RPC commands.
 472
 473     See also: test/functional/test_framework/coverage.py
 474
 475     """
 476     def __init__(self):
 477         self.dir = tempfile.mkdtemp(prefix="coverage")
 478         self.flag = '--coveragedir=%s' % self.dir
 479
 480     def report_rpc_coverage(self):
 481         """
 482         Print out RPC commands that were unexercised by tests.
 483
 484         """
 485         uncovered = self._get_uncovered_rpc_commands()
 486
 487         if uncovered:
 488             print("Uncovered RPC commands:")
 489             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 490         else:
 491             print("All RPC commands covered.")
 492
 493     def cleanup(self):
 494         return shutil.rmtree(self.dir)
 495
 496     def _get_uncovered_rpc_commands(self):
 497         """
 498         Return a set of currently untested RPC commands.
 499
 500         """
 501         # This is shared from `test/functional/test-framework/coverage.py`
 502         reference_filename = 'rpc_interface.txt'
 503         coverage_file_prefix = 'coverage.'
 504
 505         coverage_ref_filename = os.path.join(self.dir, reference_filename)
 506         coverage_filenames = set()
 507         all_cmds = set()
 508         covered_cmds = set()
 509
 510         if not os.path.isfile(coverage_ref_filename):
 511             raise RuntimeError("No coverage reference found")
 512
 513         with open(coverage_ref_filename, 'r') as f:
 514             all_cmds.update([i.strip() for i in f.readlines()])
 515
 516         for root, dirs, files in os.walk(self.dir):
 517             for filename in files:
 518                 if filename.startswith(coverage_file_prefix):
 519                     coverage_filenames.add(os.path.join(root, filename))
 520
 521         for filename in coverage_filenames:
 522             with open(filename, 'r') as f:
 523                 covered_cmds.update([i.strip() for i in f.readlines()])
 524
 525         return all_cmds - covered_cmds
 526
 527
 528 if __name__ == '__main__':
 529     main()