test/functional/test_runner.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5 """Run regression test suite.
   6
   7 This module calls down into individual test cases via subprocess. It will
   8 forward all unrecognized arguments onto the individual test scripts.
   9
  10 Functional tests are disabled on Windows by default. Use --force to run them anyway.
  11
  12 For a description of arguments recognized by test scripts, see
  13 `test/functional/test_framework/test_framework.py:BitcoinTestFramework.main`.
  14
  15 """
  16
  17 import argparse
  18 import configparser
  19 import datetime
  20 import os
  21 import time
  22 import shutil
  23 import signal
  24 import sys
  25 import subprocess
  26 import tempfile
  27 import re
  28 import logging
  29
  30 # Formatting. Default colors to empty strings.
  31 BOLD, BLUE, RED, GREY = ("", ""), ("", ""), ("", ""), ("", "")
  32 try:
  33     # Make sure python thinks it can write unicode to its stdout
  34     "\u2713".encode("utf_8").decode(sys.stdout.encoding)
  35     TICK = "✓ "
  36     CROSS = "✖ "
  37     CIRCLE = "○ "
  38 except UnicodeDecodeError:
  39     TICK = "P "
  40     CROSS = "x "
  41     CIRCLE = "o "
  42
  43 if os.name == 'posix':
  44     # primitive formatting on supported
  45     # terminal via ANSI escape sequences:
  46     BOLD = ('\033[0m', '\033[1m')
  47     BLUE = ('\033[0m', '\033[0;34m')
  48     RED = ('\033[0m', '\033[0;31m')
  49     GREY = ('\033[0m', '\033[1;30m')
  50
  51 TEST_EXIT_PASSED = 0
  52 TEST_EXIT_SKIPPED = 77
  53
  54 BASE_SCRIPTS= [
  55     # Scripts that are run by the travis build process.
  56     # Longest test should go first, to favor running tests in parallel
  57     'wallet-hd.py',
  58     'walletbackup.py',
  59     # vv Tests less than 5m vv
  60     'p2p-fullblocktest.py',
  61     'fundrawtransaction.py',
  62     'p2p-compactblocks.py',
  63     'segwit.py',
  64     # vv Tests less than 2m vv
  65     'wallet.py',
  66     'wallet-accounts.py',
  67     'p2p-segwit.py',
  68     'wallet-dump.py',
  69     'listtransactions.py',
  70     # vv Tests less than 60s vv
  71     'sendheaders.py',
  72     'zapwallettxes.py',
  73     'importmulti.py',
  74     'mempool_limit.py',
  75     'merkle_blocks.py',
  76     'receivedby.py',
  77     'abandonconflict.py',
  78     'bip68-112-113-p2p.py',
  79     'rawtransactions.py',
  80     'reindex.py',
  81     # vv Tests less than 30s vv
  82     'keypool-topup.py',
  83     'zmq_test.py',
  84     'bitcoin_cli.py',
  85     'mempool_resurrect_test.py',
  86     'txn_doublespend.py --mineblock',
  87     'txn_clone.py',
  88     'getchaintips.py',
  89     'rest.py',
  90     'mempool_spendcoinbase.py',
  91     'mempool_reorg.py',
  92     'mempool_persist.py',
  93     'multiwallet.py',
  94     'httpbasics.py',
  95     'multi_rpc.py',
  96     'proxy_test.py',
  97     'signrawtransactions.py',
  98     'disconnect_ban.py',
  99     'decodescript.py',
 100     'blockchain.py',
 101     'disablewallet.py',
 102     'net.py',
 103     'keypool.py',
 104     'p2p-mempool.py',
 105     'prioritise_transaction.py',
 106     'invalidblockrequest.py',
 107     'invalidtxrequest.py',
 108     'p2p-versionbits-warning.py',
 109     'preciousblock.py',
 110     'importprunedfunds.py',
 111     'signmessages.py',
 112     'nulldummy.py',
 113     'import-rescan.py',
 114     'mining.py',
 115     'bumpfee.py',
 116     'rpcnamedargs.py',
 117     'listsinceblock.py',
 118     'p2p-leaktests.py',
 119     'wallet-encryption.py',
 120     'bipdersig-p2p.py',
 121     'bip65-cltv-p2p.py',
 122     'uptime.py',
 123     'resendwallettransactions.py',
 124     'minchainwork.py',
 125 ]
 126
 127 EXTENDED_SCRIPTS = [
 128     # These tests are not run by the travis build process.
 129     # Longest test should go first, to favor running tests in parallel
 130     'pruning.py',
 131     # vv Tests less than 20m vv
 132     'smartfees.py',
 133     # vv Tests less than 5m vv
 134     'maxuploadtarget.py',
 135     'mempool_packages.py',
 136     'dbcrash.py',
 137     # vv Tests less than 2m vv
 138     'bip68-sequence.py',
 139     'getblocktemplate_longpoll.py',
 140     'p2p-timeouts.py',
 141     # vv Tests less than 60s vv
 142     'bip9-softforks.py',
 143     'p2p-feefilter.py',
 144     'rpcbind_test.py',
 145     # vv Tests less than 30s vv
 146     'assumevalid.py',
 147     'example_test.py',
 148     'txn_doublespend.py',
 149     'txn_clone.py --mineblock',
 150     'forknotify.py',
 151     'invalidateblock.py',
 152     'p2p-acceptblock.py',
 153     'replace-by-fee.py',
 154 ]
 155
 156 # Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
 157 ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
 158
 159 NON_SCRIPTS = [
 160     # These are python files that live in the functional tests directory, but are not test scripts.
 161     "combine_logs.py",
 162     "create_cache.py",
 163     "test_runner.py",
 164 ]
 165
 166 def main():
 167     # Parse arguments and pass through unrecognised args
 168     parser = argparse.ArgumentParser(add_help=False,
 169                                      usage='%(prog)s [test_runner.py options] [script options] [scripts]',
 170                                      description=__doc__,
 171                                      epilog='''
 172     Help text and arguments for individual test script:''',
 173                                      formatter_class=argparse.RawTextHelpFormatter)
 174     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
 175     parser.add_argument('--exclude', '-x', help='specify a comma-separated-list of scripts to exclude.')
 176     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
 177     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
 178     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
 179     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
 180     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
 181     parser.add_argument('--quiet', '-q', action='store_true', help='only print results summary and failure logs')
 182     parser.add_argument('--tmpdirprefix', '-t', default=tempfile.gettempdir(), help="Root directory for datadirs")
 183     args, unknown_args = parser.parse_known_args()
 184
 185     # args to be passed on always start with two dashes; tests are the remaining unknown args
 186     tests = [arg for arg in unknown_args if arg[:2] != "--"]
 187     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 188
 189     # Read config generated by configure.
 190     config = configparser.ConfigParser()
 191     configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini"
 192     config.read_file(open(configfile))
 193
 194     passon_args.append("--configfile=%s" % configfile)
 195
 196     # Set up logging
 197     logging_level = logging.INFO if args.quiet else logging.DEBUG
 198     logging.basicConfig(format='%(message)s', level=logging_level)
 199
 200     # Create base test directory
 201     tmpdir = "%s/bitcoin_test_runner_%s" % (args.tmpdirprefix, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
 202     os.makedirs(tmpdir)
 203
 204     logging.debug("Temporary test directory at %s" % tmpdir)
 205
 206     enable_wallet = config["components"].getboolean("ENABLE_WALLET")
 207     enable_utils = config["components"].getboolean("ENABLE_UTILS")
 208     enable_bitcoind = config["components"].getboolean("ENABLE_BITCOIND")
 209
 210     if config["environment"]["EXEEXT"] == ".exe" and not args.force:
 211         # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
 212         # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
 213         print("Tests currently disabled on Windows by default. Use --force option to enable")
 214         sys.exit(0)
 215
 216     if not (enable_wallet and enable_utils and enable_bitcoind):
 217         print("No functional tests to run. Wallet, utils, and bitcoind must all be enabled")
 218         print("Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make")
 219         sys.exit(0)
 220
 221     # Build list of tests
 222     if tests:
 223         # Individual tests have been specified. Run specified tests that exist
 224         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
 225         tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
 226         test_list = []
 227         for t in tests:
 228             if t in ALL_SCRIPTS:
 229                 test_list.append(t)
 230             else:
 231                 print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
 232     else:
 233         # No individual tests have been specified.
 234         # Run all base tests, and optionally run extended tests.
 235         test_list = BASE_SCRIPTS
 236         if args.extended:
 237             # place the EXTENDED_SCRIPTS first since the three longest ones
 238             # are there and the list is shorter
 239             test_list = EXTENDED_SCRIPTS + test_list
 240
 241     # Remove the test cases that the user has explicitly asked to exclude.
 242     if args.exclude:
 243         tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
 244         for exclude_test in tests_excl:
 245             if exclude_test in test_list:
 246                 test_list.remove(exclude_test)
 247             else:
 248                 print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
 249
 250     if not test_list:
 251         print("No valid test scripts specified. Check that your test is in one "
 252               "of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests")
 253         sys.exit(0)
 254
 255     if args.help:
 256         # Print help for test_runner.py, then print help of the first script (with args removed) and exit.
 257         parser.print_help()
 258         subprocess.check_call([(config["environment"]["SRCDIR"] + '/test/functional/' + test_list[0].split()[0])] + ['-h'])
 259         sys.exit(0)
 260
 261     check_script_list(config["environment"]["SRCDIR"])
 262
 263     if not args.keepcache:
 264         shutil.rmtree("%s/test/cache" % config["environment"]["BUILDDIR"], ignore_errors=True)
 265
 266     run_tests(test_list, config["environment"]["SRCDIR"], config["environment"]["BUILDDIR"], config["environment"]["EXEEXT"], tmpdir, args.jobs, args.coverage, passon_args)
 267
 268 def run_tests(test_list, src_dir, build_dir, exeext, tmpdir, jobs=1, enable_coverage=False, args=[]):
 269     # Warn if bitcoind is already running (unix only)
 270     try:
 271         if subprocess.check_output(["pidof", "bitcoind"]) is not None:
 272             print("%sWARNING!%s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention!" % (BOLD[1], BOLD[0]))
 273     except (OSError, subprocess.SubprocessError):
 274         pass
 275
 276     # Warn if there is a cache directory
 277     cache_dir = "%s/test/cache" % build_dir
 278     if os.path.isdir(cache_dir):
 279         print("%sWARNING!%s There is a cache directory here: %s. If tests fail unexpectedly, try deleting the cache directory." % (BOLD[1], BOLD[0], cache_dir))
 280
 281     #Set env vars
 282     if "BITCOIND" not in os.environ:
 283         os.environ["BITCOIND"] = build_dir + '/src/bitcoind' + exeext
 284         os.environ["BITCOINCLI"] = build_dir + '/src/bitcoin-cli' + exeext
 285
 286     tests_dir = src_dir + '/test/functional/'
 287
 288     flags = ["--srcdir={}/src".format(build_dir)] + args
 289     flags.append("--cachedir=%s" % cache_dir)
 290
 291     if enable_coverage:
 292         coverage = RPCCoverage()
 293         flags.append(coverage.flag)
 294         logging.debug("Initializing coverage directory at %s" % coverage.dir)
 295     else:
 296         coverage = None
 297
 298     if len(test_list) > 1 and jobs > 1:
 299         # Populate cache
 300         subprocess.check_output([tests_dir + 'create_cache.py'] + flags + ["--tmpdir=%s/cache" % tmpdir])
 301
 302     #Run Tests
 303     job_queue = TestHandler(jobs, tests_dir, tmpdir, test_list, flags)
 304     time0 = time.time()
 305     test_results = []
 306
 307     max_len_name = len(max(test_list, key=len))
 308
 309     for _ in range(len(test_list)):
 310         test_result, stdout, stderr = job_queue.get_next()
 311         test_results.append(test_result)
 312
 313         if test_result.status == "Passed":
 314             logging.debug("\n%s%s%s passed, Duration: %s s" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 315         elif test_result.status == "Skipped":
 316             logging.debug("\n%s%s%s skipped" % (BOLD[1], test_result.name, BOLD[0]))
 317         else:
 318             print("\n%s%s%s failed, Duration: %s s\n" % (BOLD[1], test_result.name, BOLD[0], test_result.time))
 319             print(BOLD[1] + 'stdout:\n' + BOLD[0] + stdout + '\n')
 320             print(BOLD[1] + 'stderr:\n' + BOLD[0] + stderr + '\n')
 321
 322     print_results(test_results, max_len_name, (int(time.time() - time0)))
 323
 324     if coverage:
 325         coverage.report_rpc_coverage()
 326
 327         logging.debug("Cleaning up coverage data")
 328         coverage.cleanup()
 329
 330     # Clear up the temp directory if all subdirectories are gone
 331     if not os.listdir(tmpdir):
 332         os.rmdir(tmpdir)
 333
 334     all_passed = all(map(lambda test_result: test_result.was_successful, test_results))
 335
 336     sys.exit(not all_passed)
 337
 338 def print_results(test_results, max_len_name, runtime):
 339     results = "\n" + BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "STATUS   ", "DURATION") + BOLD[0]
 340
 341     test_results.sort(key=lambda result: result.name.lower())
 342     all_passed = True
 343     time_sum = 0
 344
 345     for test_result in test_results:
 346         all_passed = all_passed and test_result.was_successful
 347         time_sum += test_result.time
 348         test_result.padding = max_len_name
 349         results += str(test_result)
 350
 351     status = TICK + "Passed" if all_passed else CROSS + "Failed"
 352     results += BOLD[1] + "\n%s | %s | %s s (accumulated) \n" % ("ALL".ljust(max_len_name), status.ljust(9), time_sum) + BOLD[0]
 353     results += "Runtime: %s s\n" % (runtime)
 354     print(results)
 355
 356 class TestHandler:
 357     """
 358     Trigger the test scripts passed in via the list.
 359     """
 360
 361     def __init__(self, num_tests_parallel, tests_dir, tmpdir, test_list=None, flags=None):
 362         assert(num_tests_parallel >= 1)
 363         self.num_jobs = num_tests_parallel
 364         self.tests_dir = tests_dir
 365         self.tmpdir = tmpdir
 366         self.test_list = test_list
 367         self.flags = flags
 368         self.num_running = 0
 369         # In case there is a graveyard of zombie bitcoinds, we can apply a
 370         # pseudorandom offset to hopefully jump over them.
 371         # (625 is PORT_RANGE/MAX_NODES)
 372         self.portseed_offset = int(time.time() * 1000) % 625
 373         self.jobs = []
 374
 375     def get_next(self):
 376         while self.num_running < self.num_jobs and self.test_list:
 377             # Add tests
 378             self.num_running += 1
 379             t = self.test_list.pop(0)
 380             portseed = len(self.test_list) + self.portseed_offset
 381             portseed_arg = ["--portseed={}".format(portseed)]
 382             log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16)
 383             log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16)
 384             test_argv = t.split()
 385             tmpdir = ["--tmpdir=%s/%s_%s" % (self.tmpdir, re.sub(".py$", "", test_argv[0]), portseed)]
 386             self.jobs.append((t,
 387                               time.time(),
 388                               subprocess.Popen([self.tests_dir + test_argv[0]] + test_argv[1:] + self.flags + portseed_arg + tmpdir,
 389                                                universal_newlines=True,
 390                                                stdout=log_stdout,
 391                                                stderr=log_stderr),
 392                               log_stdout,
 393                               log_stderr))
 394         if not self.jobs:
 395             raise IndexError('pop from empty list')
 396         while True:
 397             # Return first proc that finishes
 398             time.sleep(.5)
 399             for j in self.jobs:
 400                 (name, time0, proc, log_out, log_err) = j
 401                 if os.getenv('TRAVIS') == 'true' and int(time.time() - time0) > 20 * 60:
 402                     # In travis, timeout individual tests after 20 minutes (to stop tests hanging and not
 403                     # providing useful output.
 404                     proc.send_signal(signal.SIGINT)
 405                 if proc.poll() is not None:
 406                     log_out.seek(0), log_err.seek(0)
 407                     [stdout, stderr] = [l.read().decode('utf-8') for l in (log_out, log_err)]
 408                     log_out.close(), log_err.close()
 409                     if proc.returncode == TEST_EXIT_PASSED and stderr == "":
 410                         status = "Passed"
 411                     elif proc.returncode == TEST_EXIT_SKIPPED:
 412                         status = "Skipped"
 413                     else:
 414                         status = "Failed"
 415                     self.num_running -= 1
 416                     self.jobs.remove(j)
 417
 418                     return TestResult(name, status, int(time.time() - time0)), stdout, stderr
 419             print('.', end='', flush=True)
 420
 421 class TestResult():
 422     def __init__(self, name, status, time):
 423         self.name = name
 424         self.status = status
 425         self.time = time
 426         self.padding = 0
 427
 428     def __repr__(self):
 429         if self.status == "Passed":
 430             color = BLUE
 431             glyph = TICK
 432         elif self.status == "Failed":
 433             color = RED
 434             glyph = CROSS
 435         elif self.status == "Skipped":
 436             color = GREY
 437             glyph = CIRCLE
 438
 439         return color[1] + "%s | %s%s | %s s\n" % (self.name.ljust(self.padding), glyph, self.status.ljust(7), self.time) + color[0]
 440
 441     @property
 442     def was_successful(self):
 443         return self.status != "Failed"
 444
 445
 446 def check_script_list(src_dir):
 447     """Check scripts directory.
 448
 449     Check that there are no scripts in the functional tests directory which are
 450     not being run by pull-tester.py."""
 451     script_dir = src_dir + '/test/functional/'
 452     python_files = set([t for t in os.listdir(script_dir) if t[-3:] == ".py"])
 453     missed_tests = list(python_files - set(map(lambda x: x.split()[0], ALL_SCRIPTS + NON_SCRIPTS)))
 454     if len(missed_tests) != 0:
 455         print("%sWARNING!%s The following scripts are not being run: %s. Check the test lists in test_runner.py." % (BOLD[1], BOLD[0], str(missed_tests)))
 456         if os.getenv('TRAVIS') == 'true':
 457             # On travis this warning is an error to prevent merging incomplete commits into master
 458             sys.exit(1)
 459
 460 class RPCCoverage(object):
 461     """
 462     Coverage reporting utilities for test_runner.
 463
 464     Coverage calculation works by having each test script subprocess write
 465     coverage files into a particular directory. These files contain the RPC
 466     commands invoked during testing, as well as a complete listing of RPC
 467     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 468
 469     After all tests complete, the commands run are combined and diff'd against
 470     the complete list to calculate uncovered RPC commands.
 471
 472     See also: test/functional/test_framework/coverage.py
 473
 474     """
 475     def __init__(self):
 476         self.dir = tempfile.mkdtemp(prefix="coverage")
 477         self.flag = '--coveragedir=%s' % self.dir
 478
 479     def report_rpc_coverage(self):
 480         """
 481         Print out RPC commands that were unexercised by tests.
 482
 483         """
 484         uncovered = self._get_uncovered_rpc_commands()
 485
 486         if uncovered:
 487             print("Uncovered RPC commands:")
 488             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 489         else:
 490             print("All RPC commands covered.")
 491
 492     def cleanup(self):
 493         return shutil.rmtree(self.dir)
 494
 495     def _get_uncovered_rpc_commands(self):
 496         """
 497         Return a set of currently untested RPC commands.
 498
 499         """
 500         # This is shared from `test/functional/test-framework/coverage.py`
 501         reference_filename = 'rpc_interface.txt'
 502         coverage_file_prefix = 'coverage.'
 503
 504         coverage_ref_filename = os.path.join(self.dir, reference_filename)
 505         coverage_filenames = set()
 506         all_cmds = set()
 507         covered_cmds = set()
 508
 509         if not os.path.isfile(coverage_ref_filename):
 510             raise RuntimeError("No coverage reference found")
 511
 512         with open(coverage_ref_filename, 'r') as f:
 513             all_cmds.update([i.strip() for i in f.readlines()])
 514
 515         for root, dirs, files in os.walk(self.dir):
 516             for filename in files:
 517                 if filename.startswith(coverage_file_prefix):
 518                     coverage_filenames.add(os.path.join(root, filename))
 519
 520         for filename in coverage_filenames:
 521             with open(filename, 'r') as f:
 522                 covered_cmds.update([i.strip() for i in f.readlines()])
 523
 524         return all_cmds - covered_cmds
 525
 526
 527 if __name__ == '__main__':
 528     main()