test/util/bitcoin-util-test.py

   1 #!/usr/bin/env python3
   2 # Copyright 2014 BitPay Inc.
   3 # Copyright 2016-2017 The Bitcoin Core developers
   4 # Distributed under the MIT software license, see the accompanying
   5 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
   6 """Test framework for bitcoin utils.
   7
   8 Runs automatically during `make check`.
   9
  10 Can also be run manually."""
  11
  12 from __future__ import division,print_function,unicode_literals
  13
  14 import argparse
  15 import binascii
  16 try:
  17     import configparser
  18 except ImportError:
  19     import ConfigParser as configparser
  20 import difflib
  21 import json
  22 import logging
  23 import os
  24 import pprint
  25 import subprocess
  26 import sys
  27
  28 def main():
  29     config = configparser.ConfigParser()
  30     config.optionxform = str
  31     config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini")))
  32     env_conf = dict(config.items('environment'))
  33
  34     parser = argparse.ArgumentParser(description=__doc__)
  35     parser.add_argument('-v', '--verbose', action='store_true')
  36     args = parser.parse_args()
  37     verbose = args.verbose
  38
  39     if verbose:
  40         level = logging.DEBUG
  41     else:
  42         level = logging.ERROR
  43     formatter = '%(asctime)s - %(levelname)s - %(message)s'
  44     # Add the format/level to the logger
  45     logging.basicConfig(format=formatter, level=level)
  46
  47     bctester(os.path.join(env_conf["SRCDIR"], "test/util/data"), "bitcoin-util-test.json", env_conf)
  48
  49 def bctester(testDir, input_basename, buildenv):
  50     """ Loads and parses the input file, runs all tests and reports results"""
  51     input_filename = os.path.join(testDir, input_basename)
  52     raw_data = open(input_filename).read()
  53     input_data = json.loads(raw_data)
  54
  55     failed_testcases = []
  56
  57     for testObj in input_data:
  58         try:
  59             bctest(testDir, testObj, buildenv)
  60             logging.info("PASSED: " + testObj["description"])
  61         except:
  62             logging.info("FAILED: " + testObj["description"])
  63             failed_testcases.append(testObj["description"])
  64
  65     if failed_testcases:
  66         error_message = "FAILED_TESTCASES:\n"
  67         error_message += pprint.pformat(failed_testcases, width=400)
  68         logging.error(error_message)
  69         sys.exit(1)
  70     else:
  71         sys.exit(0)
  72
  73 def bctest(testDir, testObj, buildenv):
  74     """Runs a single test, comparing output and RC to expected output and RC.
  75
  76     Raises an error if input can't be read, executable fails, or output/RC
  77     are not as expected. Error is caught by bctester() and reported.
  78     """
  79     # Get the exec names and arguments
  80     execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"])
  81     execargs = testObj['args']
  82     execrun = [execprog] + execargs
  83
  84     # Read the input data (if there is any)
  85     stdinCfg = None
  86     inputData = None
  87     if "input" in testObj:
  88         filename = os.path.join(testDir, testObj["input"])
  89         inputData = open(filename).read()
  90         stdinCfg = subprocess.PIPE
  91
  92     # Read the expected output data (if there is any)
  93     outputFn = None
  94     outputData = None
  95     outputType = None
  96     if "output_cmp" in testObj:
  97         outputFn = testObj['output_cmp']
  98         outputType = os.path.splitext(outputFn)[1][1:]  # output type from file extension (determines how to compare)
  99         try:
 100             outputData = open(os.path.join(testDir, outputFn)).read()
 101         except:
 102             logging.error("Output file " + outputFn + " can not be opened")
 103             raise
 104         if not outputData:
 105             logging.error("Output data missing for " + outputFn)
 106             raise Exception
 107         if not outputType:
 108             logging.error("Output file %s does not have a file extension" % outputFn)
 109             raise Exception
 110
 111     # Run the test
 112     proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
 113     try:
 114         outs = proc.communicate(input=inputData)
 115     except OSError:
 116         logging.error("OSError, Failed to execute " + execprog)
 117         raise
 118
 119     if outputData:
 120         data_mismatch, formatting_mismatch = False, False
 121         # Parse command output and expected output
 122         try:
 123             a_parsed = parse_output(outs[0], outputType)
 124         except Exception as e:
 125             logging.error('Error parsing command output as %s: %s' % (outputType, e))
 126             raise
 127         try:
 128             b_parsed = parse_output(outputData, outputType)
 129         except Exception as e:
 130             logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
 131             raise
 132         # Compare data
 133         if a_parsed != b_parsed:
 134             logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
 135             data_mismatch = True
 136         # Compare formatting
 137         if outs[0] != outputData:
 138             error_message = "Output formatting mismatch for " + outputFn + ":\n"
 139             error_message += "".join(difflib.context_diff(outputData.splitlines(True),
 140                                                           outs[0].splitlines(True),
 141                                                           fromfile=outputFn,
 142                                                           tofile="returned"))
 143             logging.error(error_message)
 144             formatting_mismatch = True
 145
 146         assert not data_mismatch and not formatting_mismatch
 147
 148     # Compare the return code to the expected return code
 149     wantRC = 0
 150     if "return_code" in testObj:
 151         wantRC = testObj['return_code']
 152     if proc.returncode != wantRC:
 153         logging.error("Return code mismatch for " + outputFn)
 154         raise Exception
 155
 156     if "error_txt" in testObj:
 157         want_error = testObj["error_txt"]
 158         # Compare error text
 159         # TODO: ideally, we'd compare the strings exactly and also assert
 160         # That stderr is empty if no errors are expected. However, bitcoin-tx
 161         # emits DISPLAY errors when running as a windows application on
 162         # linux through wine. Just assert that the expected error text appears
 163         # somewhere in stderr.
 164         if want_error not in outs[1]:
 165             logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
 166             raise Exception
 167
 168 def parse_output(a, fmt):
 169     """Parse the output according to specified format.
 170
 171     Raise an error if the output can't be parsed."""
 172     if fmt == 'json':  # json: compare parsed data
 173         return json.loads(a)
 174     elif fmt == 'hex':  # hex: parse and compare binary data
 175         return binascii.a2b_hex(a.strip())
 176     else:
 177         raise NotImplementedError("Don't know how to compare %s" % fmt)
 178
 179 if __name__ == '__main__':
 180     main()