tools/command_tester.py

   1 #!/usr/bin/python
   2 # Copyright 2008, Google Inc.
   3 # All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31
  32 """Simple testing harness for running commands and checking expected output.
  33
  34 This harness is used instead of shell scripts to ensure windows compatibility
  35
  36 """
  37
  38
  39 # python imports
  40 import getopt
  41 import os
  42 import sys
  43
  44 # local imports
  45 import test_lib
  46
  47
  48 def Banner(message):
  49   print '=' * 70
  50   print message
  51   print '=' * 70
  52
  53
  54 def DifferentFromGolden(actual, golden, output_type, fail_msg):
  55   """Compares actual output against golden output.
  56
  57   If there are any differences, output an error message (to stdout) with
  58   appropriate banners.
  59
  60   Args:
  61     actual: actual output from the program under test, as a single
  62       string.
  63
  64     golden: expected output from the program under test, as a single
  65       string.
  66
  67     output_type: the name / title for the output type being compared.
  68       Used in banner output.
  69
  70     fail_msg:  additional failure output message, printed at the end.
  71
  72   Returns:
  73     True when there is a difference, False otherwise.
  74   """
  75
  76   diff = list(test_lib.DiffStringsIgnoringWhiteSpace(actual, golden))
  77   diff = '\n'.join(diff)
  78   if diff:
  79     Banner('Error %s diff found' % output_type)
  80     print diff
  81     Banner('Potential New Golden Output')
  82     print actual
  83     print fail_msg
  84     return True
  85   return False
  86
  87
  88 GlobalSettings = {
  89     'exit_status': 0,
  90     'osenv': None,
  91
  92     'name': None,
  93
  94     'stdin': None,
  95     'logout': None,
  96
  97     'stdout_golden': None,
  98     'stderr_golden': None,
  99     'log_golden': None,
 100
 101     'stdout_filter': None,
 102     'stderr_filter': None,
 103     'log_filter': None,
 104 }
 105
 106
 107 def ProcessOptions(argv):
 108   """Process command line options and return the unprocessed left overs."""
 109   try:
 110     opts, args = getopt.getopt(argv, '', [x + '='  for x in GlobalSettings])
 111   except getopt.GetoptError, err:
 112     print str(err)  # will print something like 'option -a not recognized'
 113     sys.exit(-1)
 114
 115   for o, a in opts:
 116     # strip the leading '--'
 117     option = o[2:]
 118     assert option in GlobalSettings
 119     if type(GlobalSettings[option]) == int:
 120       GlobalSettings[option] = int(a)
 121     else:
 122       GlobalSettings[option] = a
 123   # return the unprocessed options, i.e. the command
 124   return args
 125
 126
 127 def FailureMessage():
 128   return 'FAILED %s' % GlobalSettings['name']
 129
 130
 131 def SuccessMessage():
 132   return 'PASSED %s' % GlobalSettings['name']
 133
 134
 135 def main(argv):
 136   command = ProcessOptions(argv)
 137
 138   if not GlobalSettings['name']:
 139     GlobalSettings['name'] = command[0]
 140
 141   if GlobalSettings['osenv']:
 142     env = GlobalSettings['osenv'].split(',')
 143     for e in env:
 144       key, val = e.split('=')
 145       os.putenv(key, val)
 146
 147   if GlobalSettings['logout']:
 148     try:
 149       os.unlink(GlobalSettings['logout'])  # might not pre-exist
 150     except OSError:
 151       pass
 152
 153   stdin_data = ''
 154   if GlobalSettings['stdin']:
 155     stdin_data = open(GlobalSettings['stdin']).read()
 156
 157   Banner('running %s' % str(command))
 158   _, exit_status, failed, stdout, stderr = test_lib.RunTestWithInputOutput(
 159       command, stdin_data)
 160
 161   if exit_status != GlobalSettings['exit_status'] or failed:
 162     if failed:
 163       print 'command failed'
 164     else:
 165       print 'command returned unexpected exit status ', exit_status
 166     Banner('Stdout')
 167     print stdout
 168     Banner('Stderr')
 169     print stderr
 170     print FailureMessage()
 171     return -1
 172
 173   if GlobalSettings['stdout_golden']:
 174     stdout_golden = open(GlobalSettings['stdout_golden']).read()
 175     if GlobalSettings['stdout_filter']:
 176       stdout = test_lib.RegexpFilterLines(GlobalSettings['stdout_filter'],
 177                                           stdout)
 178     if DifferentFromGolden(stdout, stdout_golden, 'Stdout', FailureMessage()):
 179       return -1
 180
 181   if GlobalSettings['stderr_golden']:
 182     stderr_golden = open(GlobalSettings['stderr_golden']).read()
 183     if GlobalSettings['stderr_filter']:
 184       stderr = test_lib.RegexpFilterLines(GlobalSettings['stderr_filter'],
 185                                           stderr)
 186     if DifferentFromGolden(stderr, stderr_golden, 'Stderr', FailureMessage()):
 187       return -1
 188
 189   if GlobalSettings['log_golden']:
 190     log_golden = open(GlobalSettings['log_golden']).read()
 191     if GlobalSettings['log_filter']:
 192       log =  open(GlobalSettings['logout']).read()
 193       log = test_lib.RegexpFilterLines(GlobalSettings['log_filter'],
 194                                        log)
 195     if DifferentFromGolden(log, log_golden, 'Log', FailureMessage()):
 196       return -1
 197
 198   print SuccessMessage()
 199   return 0
 200
 201
 202 if __name__ == '__main__':
 203   sys.exit(main(sys.argv[1:]))