qa/pull-tester/rpc-tests.py

   1 #!/usr/bin/env python3
   2 # Copyright (c) 2014-2016 The Bitcoin Core developers
   3 # Distributed under the MIT software license, see the accompanying
   4 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   5
   6 """
   7 Run Regression Test Suite
   8
   9 This module calls down into individual test cases via subprocess. It will
  10 forward all unrecognized arguments onto the individual test scripts, other
  11 than:
  12
  13     - `-extended`: run the "extended" test suite in addition to the basic one.
  14     - `-win`: signal that this is running in a Windows environment, and we
  15       should run the tests.
  16     - `--coverage`: this generates a basic coverage report for the RPC
  17       interface.
  18
  19 For a description of arguments recognized by test scripts, see
  20 `qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`.
  21
  22 """
  23
  24 import os
  25 import time
  26 import shutil
  27 import sys
  28 import subprocess
  29 import tempfile
  30 import re
  31
  32 sys.path.append("qa/pull-tester/")
  33 from tests_config import *
  34
  35 BOLD = ("","")
  36 if os.name == 'posix':
  37     # primitive formatting on supported
  38     # terminal via ANSI escape sequences:
  39     BOLD = ('\033[0m', '\033[1m')
  40
  41 RPC_TESTS_DIR = SRCDIR + '/qa/rpc-tests/'
  42
  43 #If imported values are not defined then set to zero (or disabled)
  44 if 'ENABLE_WALLET' not in vars():
  45     ENABLE_WALLET=0
  46 if 'ENABLE_BITCOIND' not in vars():
  47     ENABLE_BITCOIND=0
  48 if 'ENABLE_UTILS' not in vars():
  49     ENABLE_UTILS=0
  50 if 'ENABLE_ZMQ' not in vars():
  51     ENABLE_ZMQ=0
  52
  53 ENABLE_COVERAGE=0
  54
  55 #Create a set to store arguments and create the passon string
  56 opts = set()
  57 passon_args = []
  58 PASSON_REGEX = re.compile("^--")
  59 PARALLEL_REGEX = re.compile('^-parallel=')
  60
  61 print_help = False
  62 run_parallel = 4
  63
  64 for arg in sys.argv[1:]:
  65     if arg == "--help" or arg == "-h" or arg == "-?":
  66         print_help = True
  67         break
  68     if arg == '--coverage':
  69         ENABLE_COVERAGE = 1
  70     elif PASSON_REGEX.match(arg):
  71         passon_args.append(arg)
  72     elif PARALLEL_REGEX.match(arg):
  73         run_parallel = int(arg.split(sep='=', maxsplit=1)[1])
  74     else:
  75         opts.add(arg)
  76
  77 #Set env vars
  78 if "BITCOIND" not in os.environ:
  79     os.environ["BITCOIND"] = BUILDDIR + '/src/bitcoind' + EXEEXT
  80
  81 if EXEEXT == ".exe" and "-win" not in opts:
  82     # https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
  83     # https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
  84     print("Win tests currently disabled by default.  Use -win option to enable")
  85     sys.exit(0)
  86
  87 if not (ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1):
  88     print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled")
  89     sys.exit(0)
  90
  91 # python3-zmq may not be installed. Handle this gracefully and with some helpful info
  92 if ENABLE_ZMQ:
  93     try:
  94         import zmq
  95     except ImportError:
  96         print("ERROR: \"import zmq\" failed. Set ENABLE_ZMQ=0 or "
  97               "to run zmq tests, see dependency info in /qa/README.md.")
  98         # ENABLE_ZMQ=0
  99         raise
 100
 101 testScripts = [
 102     # longest test should go first, to favor running tests in parallel
 103     'p2p-fullblocktest.py',
 104     'walletbackup.py',
 105     'bip68-112-113-p2p.py',
 106     'wallet.py',
 107     'wallet-accounts.py',
 108     'wallet-hd.py',
 109     'wallet-dump.py',
 110     'listtransactions.py',
 111     'receivedby.py',
 112     'mempool_resurrect_test.py',
 113     'txn_doublespend.py --mineblock',
 114     'p2p-segwit.py',
 115     'segwit.py',
 116     'txn_clone.py',
 117     'getchaintips.py',
 118     'rawtransactions.py',
 119     'rest.py',
 120     'mempool_spendcoinbase.py',
 121     'mempool_reorg.py',
 122     'mempool_limit.py',
 123     'httpbasics.py',
 124     'multi_rpc.py',
 125     'zapwallettxes.py',
 126     'proxy_test.py',
 127     'merkle_blocks.py',
 128     'fundrawtransaction.py',
 129     'signrawtransactions.py',
 130     'nodehandling.py',
 131     'reindex.py',
 132     'decodescript.py',
 133     'blockchain.py',
 134     'disablewallet.py',
 135     'sendheaders.py',
 136     'keypool.py',
 137     'p2p-mempool.py',
 138     'prioritise_transaction.py',
 139     'invalidblockrequest.py',
 140     'invalidtxrequest.py',
 141     'abandonconflict.py',
 142     'p2p-versionbits-warning.py',
 143     'importprunedfunds.py',
 144     'signmessages.py',
 145     'p2p-compactblocks.py',
 146 ]
 147 if ENABLE_ZMQ:
 148     testScripts.append('zmq_test.py')
 149
 150 testScriptsExt = [
 151     'bip9-softforks.py',
 152     'bip65-cltv.py',
 153     'bip65-cltv-p2p.py',
 154     'bip68-sequence.py',
 155     'bipdersig-p2p.py',
 156     'bipdersig.py',
 157     'getblocktemplate_longpoll.py',
 158     'getblocktemplate_proposals.py',
 159     'txn_doublespend.py',
 160     'txn_clone.py --mineblock',
 161     'forknotify.py',
 162     'invalidateblock.py',
 163     'rpcbind_test.py',
 164     'smartfees.py',
 165     'maxblocksinflight.py',
 166     'p2p-acceptblock.py',
 167     'mempool_packages.py',
 168     'maxuploadtarget.py',
 169     'replace-by-fee.py',
 170     'p2p-feefilter.py',
 171     'pruning.py', # leave pruning last as it takes a REALLY long time
 172 ]
 173
 174
 175 def runtests():
 176     test_list = []
 177     if '-extended' in opts:
 178         test_list = testScripts + testScriptsExt
 179     elif len(opts) == 0 or (len(opts) == 1 and "-win" in opts):
 180         test_list = testScripts
 181     else:
 182         for t in testScripts + testScriptsExt:
 183             if t in opts or re.sub(".py$", "", t) in opts:
 184                 test_list.append(t)
 185
 186     if print_help:
 187         # Only print help of the first script and exit
 188         subprocess.check_call((RPC_TESTS_DIR + test_list[0]).split() + ['-h'])
 189         sys.exit(0)
 190
 191     coverage = None
 192
 193     if ENABLE_COVERAGE:
 194         coverage = RPCCoverage()
 195         print("Initializing coverage directory at %s\n" % coverage.dir)
 196     flags = ["--srcdir=%s/src" % BUILDDIR] + passon_args
 197     flags.append("--cachedir=%s/qa/cache" % BUILDDIR)
 198     if coverage:
 199         flags.append(coverage.flag)
 200
 201     if len(test_list) > 1 and run_parallel > 1:
 202         # Populate cache
 203         subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + flags)
 204
 205     #Run Tests
 206     max_len_name = len(max(test_list, key=len))
 207     time_sum = 0
 208     time0 = time.time()
 209     job_queue = RPCTestHandler(run_parallel, test_list, flags)
 210     results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0]
 211     all_passed = True
 212     for _ in range(len(test_list)):
 213         (name, stdout, stderr, passed, duration) = job_queue.get_next()
 214         all_passed = all_passed and passed
 215         time_sum += duration
 216
 217         print('\n' + BOLD[1] + name + BOLD[0] + ":")
 218         print(stdout)
 219         print('stderr:\n' if not stderr == '' else '', stderr)
 220         results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration)
 221         print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration))
 222     results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]
 223     print(results)
 224     print("\nRuntime: %s s" % (int(time.time() - time0)))
 225
 226     if coverage:
 227         coverage.report_rpc_coverage()
 228
 229         print("Cleaning up coverage data")
 230         coverage.cleanup()
 231
 232     sys.exit(not all_passed)
 233
 234
 235 class RPCTestHandler:
 236     """
 237     Trigger the testscrips passed in via the list.
 238     """
 239
 240     def __init__(self, num_tests_parallel, test_list=None, flags=None):
 241         assert(num_tests_parallel >= 1)
 242         self.num_jobs = num_tests_parallel
 243         self.test_list = test_list
 244         self.flags = flags
 245         self.num_running = 0
 246         self.jobs = []
 247
 248     def get_next(self):
 249         while self.num_running < self.num_jobs and self.test_list:
 250             # Add tests
 251             self.num_running += 1
 252             t = self.test_list.pop(0)
 253             port_seed = ["--portseed=%s" % len(self.test_list)]
 254             self.jobs.append((t,
 255                               time.time(),
 256                               subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags + port_seed,
 257                                                universal_newlines=True,
 258                                                stdout=subprocess.PIPE,
 259                                                stderr=subprocess.PIPE)))
 260         if not self.jobs:
 261             raise IndexError('pop from empty list')
 262         while True:
 263             # Return first proc that finishes
 264             time.sleep(.5)
 265             for j in self.jobs:
 266                 (name, time0, proc) = j
 267                 if proc.poll() is not None:
 268                     (stdout, stderr) = proc.communicate(timeout=3)
 269                     passed = stderr == "" and proc.returncode == 0
 270                     self.num_running -= 1
 271                     self.jobs.remove(j)
 272                     return name, stdout, stderr, passed, int(time.time() - time0)
 273             print('.', end='', flush=True)
 274
 275
 276 class RPCCoverage(object):
 277     """
 278     Coverage reporting utilities for pull-tester.
 279
 280     Coverage calculation works by having each test script subprocess write
 281     coverage files into a particular directory. These files contain the RPC
 282     commands invoked during testing, as well as a complete listing of RPC
 283     commands per `bitcoin-cli help` (`rpc_interface.txt`).
 284
 285     After all tests complete, the commands run are combined and diff'd against
 286     the complete list to calculate uncovered RPC commands.
 287
 288     See also: qa/rpc-tests/test_framework/coverage.py
 289
 290     """
 291     def __init__(self):
 292         self.dir = tempfile.mkdtemp(prefix="coverage")
 293         self.flag = '--coveragedir=%s' % self.dir
 294
 295     def report_rpc_coverage(self):
 296         """
 297         Print out RPC commands that were unexercised by tests.
 298
 299         """
 300         uncovered = self._get_uncovered_rpc_commands()
 301
 302         if uncovered:
 303             print("Uncovered RPC commands:")
 304             print("".join(("  - %s\n" % i) for i in sorted(uncovered)))
 305         else:
 306             print("All RPC commands covered.")
 307
 308     def cleanup(self):
 309         return shutil.rmtree(self.dir)
 310
 311     def _get_uncovered_rpc_commands(self):
 312         """
 313         Return a set of currently untested RPC commands.
 314
 315         """
 316         # This is shared from `qa/rpc-tests/test-framework/coverage.py`
 317         REFERENCE_FILENAME = 'rpc_interface.txt'
 318         COVERAGE_FILE_PREFIX = 'coverage.'
 319
 320         coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME)
 321         coverage_filenames = set()
 322         all_cmds = set()
 323         covered_cmds = set()
 324
 325         if not os.path.isfile(coverage_ref_filename):
 326             raise RuntimeError("No coverage reference found")
 327
 328         with open(coverage_ref_filename, 'r') as f:
 329             all_cmds.update([i.strip() for i in f.readlines()])
 330
 331         for root, dirs, files in os.walk(self.dir):
 332             for filename in files:
 333                 if filename.startswith(COVERAGE_FILE_PREFIX):
 334                     coverage_filenames.add(os.path.join(root, filename))
 335
 336         for filename in coverage_filenames:
 337             with open(filename, 'r') as f:
 338                 covered_cmds.update([i.strip() for i in f.readlines()])
 339
 340         return all_cmds - covered_cmds
 341
 342
 343 if __name__ == '__main__':
 344     runtests()