python/wvtest.py

   1 #!/usr/bin/env python
   2 #
   3 # WvTest:
   4 #   Copyright (C)2007-2012 Versabanq Innovations Inc. and contributors.
   5 #       Licensed under the GNU Library General Public License, version 2.
   6 #       See the included file named LICENSE for license information.
   7 #       You can get wvtest from: http://github.com/apenwarr/wvtest
   8 #
   9 import atexit
  10 import inspect
  11 import os
  12 import re
  13 import sys
  14 import traceback
  15
  16 # NOTE
  17 # Why do we do we need the "!= main" check?  Because if you run
  18 # wvtest.py as a main program and it imports your test files, then
  19 # those test files will try to import the wvtest module recursively.
  20 # That actually *works* fine, because we don't run this main program
  21 # when we're imported as a module.  But you end up with two separate
  22 # wvtest modules, the one that gets imported, and the one that's the
  23 # main program.  Each of them would have duplicated global variables
  24 # (most importantly, wvtest._registered), and so screwy things could
  25 # happen.  Thus, we make the main program module *totally* different
  26 # from the imported module.  Then we import wvtest (the module) into
  27 # wvtest (the main program) here and make sure to refer to the right
  28 # versions of global variables.
  29 #
  30 # All this is done just so that wvtest.py can be a single file that's
  31 # easy to import into your own applications.
  32 if __name__ != '__main__':   # we're imported as a module
  33     _registered = []
  34     _tests = 0
  35     _fails = 0
  36     _xpasses = 0
  37     _xfails = 0
  38     _skips = 0
  39
  40     def wvtest(func, innerfunc=None):
  41         """ Use this decorator (@wvtest) in front of any function you want to
  42             run as part of the unit test suite.  Then run:
  43                 python wvtest.py path/to/yourtest.py [other test.py files...]
  44             to run all the @wvtest functions in the given file(s).
  45         """
  46         _registered.append((func, innerfunc or func))
  47         return func
  48
  49     def wvskip(func):
  50         """ Use this decorator (@wvskip) to mark test as skipped """
  51         # ensure original func is removed from runtests
  52         try:
  53             _registered.remove((func, func))
  54         except ValueError:
  55             pass
  56
  57         tb = traceback.extract_stack()[-2]
  58         def skipper():
  59             _result(func.func_name, tb, 'skip ok')
  60
  61         # minimal tweaks so that inspect thinks skipper is from func place
  62         skipper.func_name = func.func_name
  63         skipper.__module__ = func.__module__
  64
  65         _registered.append((skipper, skipper))
  66
  67     def _result(msg, tb, code):
  68         global _tests, _fails, _xpasses, _xfails, _skips
  69         _tests += 1
  70         if code == 'ok':
  71             # _passes +=1
  72             pass
  73         elif code == 'xpass ok':
  74             _xpasses += 1
  75         elif code == 'xfail ok':
  76             _xfails += 1
  77         elif code == 'skip ok':
  78             _skips += 1
  79         else:
  80             _fails += 1
  81         (filename, line, func, text) = tb
  82         filename = os.path.basename(filename)
  83         msg = re.sub(r'\s+', ' ', str(msg))
  84         sys.stderr.flush()
  85         print '! %-70s %s' % ('%s:%-4d %s' % (filename, line, msg),
  86                               code)
  87         sys.stdout.flush()
  88
  89     def _check(cond, msg, xdepth):
  90         tb = traceback.extract_stack()[-3 - xdepth]
  91         if cond:
  92             _result(msg, tb, 'ok')
  93         else:
  94             _result(msg, tb, 'FAILED')
  95         return cond
  96
  97     def _code(xdepth):
  98         (filename, line, func, text) = traceback.extract_stack()[-3 - xdepth]
  99         text = re.sub(r'^[\w\.]+\((.*)\)(\s*#.*)?$', r'\1', str(text));
 100         return text
 101
 102
 103     def WVPASS(cond = True, xdepth = 0):
 104         ''' Counts a test failure unless cond is true. '''
 105         return _check(cond, _code(xdepth), xdepth)
 106
 107     def WVFAIL(cond = True, xdepth = 0):
 108         ''' Counts a test failure  unless cond is false. '''
 109         return _check(not cond, 'NOT(%s)' % _code(xdepth), xdepth)
 110
 111     def WVPASSIS(a, b, xdepth = 0):
 112         ''' Counts a test failure unless a is b. '''
 113         return _check(a is b, '%s is %s' % (repr(a), repr(b)), xdepth)
 114
 115     def WVPASSISNOT(a, b, xdepth = 0):
 116         ''' Counts a test failure unless a is not b. '''
 117         return _check(a is not b, '%s is not %s' % (repr(a), repr(b)), xdepth)
 118
 119     def WVXFAIL(cond, xdepth = 0):
 120         ''' Counts a known/fixed test failure if cond is false/true '''
 121         tb = traceback.extract_stack()[-2]
 122         res = cond and 'xpass ok' or 'xfail ok'
 123         _result(_code(xdepth), tb, res)
 124
 125     def WVPASSEQ(a, b, xdepth = 0):
 126         ''' Counts a test failure unless a == b. '''
 127         return _check(a == b, '%s == %s' % (repr(a), repr(b)), xdepth)
 128
 129     def WVPASSNE(a, b, xdepth = 0):
 130         ''' Counts a test failure unless a != b. '''
 131         return _check(a != b, '%s != %s' % (repr(a), repr(b)), xdepth)
 132
 133     def WVPASSLT(a, b, xdepth = 0):
 134         ''' Counts a test failure unless a < b. '''
 135         return _check(a < b, '%s < %s' % (repr(a), repr(b)), xdepth)
 136
 137     def WVPASSLE(a, b, xdepth = 0):
 138         ''' Counts a test failure unless a <= b. '''
 139         return _check(a <= b, '%s <= %s' % (repr(a), repr(b)), xdepth)
 140
 141     def WVPASSGT(a, b, xdepth = 0):
 142         ''' Counts a test failure unless a > b. '''
 143         return _check(a > b, '%s > %s' % (repr(a), repr(b)), xdepth)
 144
 145     def WVPASSGE(a, b, xdepth = 0):
 146         ''' Counts a test failure unless a >= b. '''
 147         return _check(a >= b, '%s >= %s' % (repr(a), repr(b)), xdepth)
 148
 149     def WVPASSNEAR(a, b, places = 7, delta = None, xdepth = 0):
 150         ''' Counts a test failure unless a ~= b. '''
 151         if delta:
 152             return _check(abs(a - b) <= abs(delta),
 153                           '%s ~= %s' % (repr(a), repr(b)), xdepth)
 154         else:
 155             return _check(round(a, places) == round(b, places),
 156                           '%s ~= %s' % (repr(a), repr(b)), xdepth)
 157
 158     def WVPASSFAR(a, b, places = 7, delta = None, xdepth = 0):
 159         ''' Counts a test failure unless a ~!= b. '''
 160         if delta:
 161             return _check(abs(a - b) > abs(delta),
 162                           '%s ~= %s' % (repr(a), repr(b)), xdepth)
 163         else:
 164             return _check(round(a, places) != round(b, places),
 165                           '%s ~= %s' % (repr(a), repr(b)), xdepth)
 166
 167     def _except_report(cond, code, xdepth):
 168         return _check(cond, 'EXCEPT(%s)' % code, xdepth + 1)
 169
 170     class _ExceptWrapper(object):
 171         def __init__(self, etype, xdepth):
 172             self.etype = etype
 173             self.xdepth = xdepth
 174             self.code = None
 175
 176         def __enter__(self):
 177           self.code = _code(self.xdepth)
 178
 179         def __exit__(self, etype, value, traceback):
 180             if etype == self.etype:
 181                 _except_report(True, self.code, self.xdepth)
 182                 return 1  # success, got the expected exception
 183             elif etype is None:
 184                 _except_report(False, self.code, self.xdepth)
 185                 return 0
 186             else:
 187                 _except_report(False, self.code, self.xdepth)
 188
 189     def _WVEXCEPT(etype, xdepth, func, *args, **kwargs):
 190         if func:
 191             code = _code(xdepth + 1)
 192             try:
 193                 func(*args, **kwargs)
 194             except etype, e:
 195                 return _except_report(True, code, xdepth + 1)
 196             except:
 197                 _except_report(False, code, xdepth + 1)
 198                 raise
 199             else:
 200                 return _except_report(False, code, xdepth + 1)
 201         else:
 202             return _ExceptWrapper(etype, xdepth)
 203
 204     def WVEXCEPT(etype, func=None, *args, **kwargs):
 205         ''' Counts a test failure unless func throws an 'etype' exception.
 206             You have to spell out the function name and arguments, rather than
 207             calling the function yourself, so that WVEXCEPT can run before
 208             your test code throws an exception.
 209         '''
 210         return _WVEXCEPT(etype, 0, func, *args, **kwargs)
 211
 212
 213     def _check_unfinished():
 214         if _registered:
 215             for func, innerfunc in _registered:
 216                 print 'WARNING: not run: %r' % (innerfunc,)
 217             WVFAIL('wvtest_main() not called')
 218         if _fails:
 219             sys.exit(1)
 220
 221     atexit.register(_check_unfinished)
 222
 223
 224 def _run_in_chdir(path, func, *args, **kwargs):
 225     oldwd = os.getcwd()
 226     oldpath = sys.path
 227     try:
 228         if path: os.chdir(path)
 229         sys.path += [path, os.path.split(path)[0]]
 230         return func(*args, **kwargs)
 231     finally:
 232         os.chdir(oldwd)
 233         sys.path = oldpath
 234
 235
 236 def _runtest(fname, f, innerfunc):
 237     import wvtest as _wvtestmod
 238     mod = inspect.getmodule(innerfunc)
 239     relpath = os.path.relpath(mod.__file__, os.getcwd()).replace('.pyc', '.py')
 240     print
 241     print 'Testing "%s" in %s:' % (fname, relpath)
 242     sys.stdout.flush()
 243     try:
 244         _run_in_chdir(os.path.split(mod.__file__)[0], f)
 245     except Exception, e:
 246         print
 247         print traceback.format_exc()
 248         tb = sys.exc_info()[2]
 249         _wvtestmod._result(repr(e), traceback.extract_tb(tb)[-1], 'EXCEPTION')
 250
 251
 252 def _run_registered_tests():
 253     import wvtest as _wvtestmod
 254     while _wvtestmod._registered:
 255         func, innerfunc = _wvtestmod._registered.pop(0)
 256         _runtest(innerfunc.func_name, func, innerfunc)
 257         print
 258
 259
 260 def wvtest_main(extra_testfiles=[]):
 261     import wvtest as _wvtestmod
 262     _run_registered_tests()
 263     for modname in extra_testfiles:
 264         if not os.path.exists(modname):
 265             print 'Skipping: %s' % modname
 266             continue
 267         if modname.endswith('.py'):
 268             modname = modname[:-3]
 269         print 'Importing: %s' % modname
 270         path, mod = os.path.split(os.path.abspath(modname))
 271         nicename = modname.replace(os.path.sep, '.')
 272         while nicename.startswith('.'):
 273             nicename = modname[1:]
 274         _run_in_chdir(path, __import__, nicename, None, None, [])
 275         _run_registered_tests()
 276     print
 277     print 'WvTest: %d tests, %d failures.' % (_wvtestmod._tests,
 278                                               _wvtestmod._fails)
 279     print 'WvTest: %d tests skipped, %d known breakages, %d fixed breakages' % (
 280             _wvtestmod._skips, _wvtestmod._xfails, _wvtestmod._xpasses)
 281
 282
 283 if __name__ == '__main__':
 284     import wvtest as _wvtestmod
 285     sys.modules['wvtest'] = _wvtestmod
 286     sys.modules['wvtest.wvtest'] = _wvtestmod
 287     wvtest_main(sys.argv[1:])