svntest/verify.py

   1 #
   2 #  verify.py:  routines that handle comparison and display of expected
   3 #              vs. actual output
   4 #
   5 #  Subversion is a tool for revision control.
   6 #  See http://subversion.tigris.org for more information.
   7 #
   8 # ====================================================================
   9 #    Licensed to the Apache Software Foundation (ASF) under one
  10 #    or more contributor license agreements.  See the NOTICE file
  11 #    distributed with this work for additional information
  12 #    regarding copyright ownership.  The ASF licenses this file
  13 #    to you under the Apache License, Version 2.0 (the
  14 #    "License"); you may not use this file except in compliance
  15 #    with the License.  You may obtain a copy of the License at
  16 #
  17 #      http://www.apache.org/licenses/LICENSE-2.0
  18 #
  19 #    Unless required by applicable law or agreed to in writing,
  20 #    software distributed under the License is distributed on an
  21 #    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  22 #    KIND, either express or implied.  See the License for the
  23 #    specific language governing permissions and limitations
  24 #    under the License.
  25 ######################################################################
  26
  27 import re, sys
  28 from difflib import unified_diff
  29
  30 import svntest
  31
  32
  33 ######################################################################
  34 # Exception types
  35
  36 class SVNUnexpectedOutput(svntest.Failure):
  37   """Exception raised if an invocation of svn results in unexpected
  38   output of any kind."""
  39   pass
  40
  41 class SVNUnexpectedStdout(SVNUnexpectedOutput):
  42   """Exception raised if an invocation of svn results in unexpected
  43   output on STDOUT."""
  44   pass
  45
  46 class SVNUnexpectedStderr(SVNUnexpectedOutput):
  47   """Exception raised if an invocation of svn results in unexpected
  48   output on STDERR."""
  49   pass
  50
  51 class SVNExpectedStdout(SVNUnexpectedOutput):
  52   """Exception raised if an invocation of svn results in no output on
  53   STDOUT when output was expected."""
  54   pass
  55
  56 class SVNExpectedStderr(SVNUnexpectedOutput):
  57   """Exception raised if an invocation of svn results in no output on
  58   STDERR when output was expected."""
  59   pass
  60
  61 class SVNUnexpectedExitCode(SVNUnexpectedOutput):
  62   """Exception raised if an invocation of svn exits with a value other
  63   than what was expected."""
  64   pass
  65
  66 class SVNIncorrectDatatype(SVNUnexpectedOutput):
  67   """Exception raised if invalid input is passed to the
  68   run_and_verify_* API"""
  69   pass
  70
  71
  72 ######################################################################
  73 # Comparison of expected vs. actual output
  74
  75 def createExpectedOutput(expected, output_type, match_all=True):
  76   """Return EXPECTED, promoted to an ExpectedOutput instance if not
  77   None.  Raise SVNIncorrectDatatype if the data type of EXPECTED is
  78   not handled."""
  79   if isinstance(expected, list):
  80     expected = ExpectedOutput(expected)
  81   elif isinstance(expected, str):
  82     expected = RegexOutput(expected, match_all)
  83   elif expected is AnyOutput:
  84     expected = AnyOutput()
  85   elif expected is not None and not isinstance(expected, ExpectedOutput):
  86     raise SVNIncorrectDatatype("Unexpected type for '%s' data" % output_type)
  87   return expected
  88
  89 class ExpectedOutput:
  90   """Contains expected output, and performs comparisons."""
  91
  92   is_regex = False
  93   is_unordered = False
  94
  95   def __init__(self, output, match_all=True):
  96     """Initialize the expected output to OUTPUT which is a string, or a list
  97     of strings, or None meaning an empty list. If MATCH_ALL is True, the
  98     expected strings will be matched with the actual strings, one-to-one, in
  99     the same order. If False, they will be matched with a subset of the
 100     actual strings, one-to-one, in the same order, ignoring any other actual
 101     strings among the matching ones."""
 102     self.output = output
 103     self.match_all = match_all
 104
 105   def __str__(self):
 106     return str(self.output)
 107
 108   def __cmp__(self, other):
 109     raise 'badness'
 110
 111   def matches(self, other, except_re=None):
 112     """Return whether SELF.output matches OTHER (which may be a list
 113     of newline-terminated lines, or a single string).  Either value
 114     may be None."""
 115     if self.output is None:
 116       expected = []
 117     else:
 118       expected = self.output
 119     if other is None:
 120       actual = []
 121     else:
 122       actual = other
 123
 124     if not isinstance(actual, list):
 125       actual = [actual]
 126     if not isinstance(expected, list):
 127       expected = [expected]
 128
 129     if except_re:
 130       return self.matches_except(expected, actual, except_re)
 131     else:
 132       return self.is_equivalent_list(expected, actual)
 133
 134   def matches_except(self, expected, actual, except_re):
 135     "Return whether EXPECTED and ACTUAL match except for except_re."
 136     if not self.is_regex:
 137       i_expected = 0
 138       i_actual = 0
 139       while i_expected < len(expected) and i_actual < len(actual):
 140         if re.match(except_re, actual[i_actual]):
 141           i_actual += 1
 142         elif re.match(except_re, expected[i_expected]):
 143           i_expected += 1
 144         elif expected[i_expected] == actual[i_actual]:
 145           i_expected += 1
 146           i_actual += 1
 147         else:
 148           return False
 149       if i_expected == len(expected) and i_actual == len(actual):
 150             return True
 151       return False
 152     else:
 153       raise Exception("is_regex and except_re are mutually exclusive")
 154
 155   def is_equivalent_list(self, expected, actual):
 156     "Return whether EXPECTED and ACTUAL are equivalent."
 157     if not self.is_regex:
 158       if self.match_all:
 159         # The EXPECTED lines must match the ACTUAL lines, one-to-one, in
 160         # the same order.
 161         return expected == actual
 162
 163       # The EXPECTED lines must match a subset of the ACTUAL lines,
 164       # one-to-one, in the same order, with zero or more other ACTUAL
 165       # lines interspersed among the matching ACTUAL lines.
 166       i_expected = 0
 167       for actual_line in actual:
 168         if expected[i_expected] == actual_line:
 169           i_expected += 1
 170           if i_expected == len(expected):
 171             return True
 172       return False
 173
 174     expected_re = expected[0]
 175     # If we want to check that every line matches the regexp
 176     # assume they all match and look for any that don't.  If
 177     # only one line matching the regexp is enough, assume none
 178     # match and look for even one that does.
 179     if self.match_all:
 180       all_lines_match_re = True
 181     else:
 182       all_lines_match_re = False
 183
 184     # If a regex was provided assume that we actually require
 185     # some output. Fail if we don't have any.
 186     if len(actual) == 0:
 187       return False
 188
 189     for actual_line in actual:
 190       if self.match_all:
 191         if not re.match(expected_re, actual_line):
 192           return False
 193       else:
 194         # As soon an actual_line matches something, then we're good.
 195         if re.match(expected_re, actual_line):
 196           return True
 197
 198     return all_lines_match_re
 199
 200   def display_differences(self, message, label, actual):
 201     """Delegate to the display_lines() routine with the appropriate
 202     args.  MESSAGE is ignored if None."""
 203     display_lines(message, label, self.output, actual,
 204                   self.is_regex, self.is_unordered)
 205
 206
 207 class AnyOutput(ExpectedOutput):
 208   def __init__(self):
 209     ExpectedOutput.__init__(self, None, False)
 210
 211   def is_equivalent_list(self, ignored, actual):
 212     if len(actual) == 0:
 213       # No actual output. No match.
 214       return False
 215
 216     for line in actual:
 217       # If any line has some text, then there is output, so we match.
 218       if line:
 219         return True
 220
 221     # We did not find a line with text. No match.
 222     return False
 223
 224   def display_differences(self, message, label, actual):
 225     if message:
 226       print(message)
 227
 228
 229 class RegexOutput(ExpectedOutput):
 230   is_regex = True
 231
 232
 233 class UnorderedOutput(ExpectedOutput):
 234   """Marks unordered output, and performs comparisons."""
 235
 236   is_unordered = True
 237
 238   def __cmp__(self, other):
 239     raise 'badness'
 240
 241   def is_equivalent_list(self, expected, actual):
 242     "Disregard the order of ACTUAL lines during comparison."
 243
 244     e_set = set(expected)
 245     a_set = set(actual)
 246
 247     if self.match_all:
 248       if len(e_set) != len(a_set):
 249         return False
 250       if self.is_regex:
 251         for expect_re in e_set:
 252           for actual_line in a_set:
 253             if re.match(expect_re, actual_line):
 254               a_set.remove(actual_line)
 255               break
 256           else:
 257             # One of the regexes was not found
 258             return False
 259         return True
 260
 261       # All expected lines must be in the output.
 262       return e_set == a_set
 263
 264     if self.is_regex:
 265       # If any of the expected regexes are in the output, then we match.
 266       for expect_re in e_set:
 267         for actual_line in a_set:
 268           if re.match(expect_re, actual_line):
 269             return True
 270       return False
 271
 272     # If any of the expected lines are in the output, then we match.
 273     return len(e_set.intersection(a_set)) > 0
 274
 275
 276 class UnorderedRegexOutput(UnorderedOutput, RegexOutput):
 277   is_regex = True
 278   is_unordered = True
 279
 280
 281 ######################################################################
 282 # Displaying expected and actual output
 283
 284 def display_trees(message, label, expected, actual):
 285   'Print two trees, expected and actual.'
 286   if message is not None:
 287     print(message)
 288   if expected is not None:
 289     print('EXPECTED %s:' % label)
 290     svntest.tree.dump_tree(expected)
 291   if actual is not None:
 292     print('ACTUAL %s:' % label)
 293     svntest.tree.dump_tree(actual)
 294
 295
 296 def display_lines(message, label, expected, actual, expected_is_regexp=None,
 297                   expected_is_unordered=None):
 298   """Print MESSAGE, unless it is None, then print EXPECTED (labeled
 299   with LABEL) followed by ACTUAL (also labeled with LABEL).
 300   Both EXPECTED and ACTUAL may be strings or lists of strings."""
 301   if message is not None:
 302     print(message)
 303   if expected is not None:
 304     output = 'EXPECTED %s' % label
 305     if expected_is_regexp:
 306       output += ' (regexp)'
 307       expected = [expected + '\n']
 308     if expected_is_unordered:
 309       output += ' (unordered)'
 310     output += ':'
 311     print(output)
 312     for x in expected:
 313       sys.stdout.write(x)
 314   if actual is not None:
 315     print('ACTUAL %s:' % label)
 316     for x in actual:
 317       sys.stdout.write(x)
 318
 319   # Additionally print unified diff
 320   if not expected_is_regexp:
 321     print('DIFF ' + ' '.join(output.split(' ')[1:]))
 322
 323     if type(expected) is str:
 324       expected = [expected]
 325
 326     if type(actual) is str:
 327       actual = [actual]
 328
 329     for x in unified_diff(expected, actual,
 330                           fromfile="EXPECTED %s" % label,
 331                           tofile="ACTUAL %s" % label):
 332       sys.stdout.write(x)
 333
 334 def compare_and_display_lines(message, label, expected, actual,
 335                               raisable=None, except_re=None):
 336   """Compare two sets of output lines, and print them if they differ,
 337   preceded by MESSAGE iff not None.  EXPECTED may be an instance of
 338   ExpectedOutput (and if not, it is wrapped as such).  RAISABLE is an
 339   exception class, an instance of which is thrown if ACTUAL doesn't
 340   match EXPECTED."""
 341   if raisable is None:
 342     raisable = svntest.main.SVNLineUnequal
 343   ### It'd be nicer to use createExpectedOutput() here, but its
 344   ### semantics don't match all current consumers of this function.
 345   if not isinstance(expected, ExpectedOutput):
 346     expected = ExpectedOutput(expected)
 347
 348   if isinstance(actual, str):
 349     actual = [actual]
 350   actual = [line for line in actual if not line.startswith('DBG:')]
 351
 352   if not expected.matches(actual, except_re):
 353     expected.display_differences(message, label, actual)
 354     raise raisable
 355
 356 def verify_outputs(message, actual_stdout, actual_stderr,
 357                    expected_stdout, expected_stderr, all_stdout=True):
 358   """Compare and display expected vs. actual stderr and stdout lines:
 359   if they don't match, print the difference (preceded by MESSAGE iff
 360   not None) and raise an exception.
 361
 362   If EXPECTED_STDERR or EXPECTED_STDOUT is a string the string is
 363   interpreted as a regular expression.  For EXPECTED_STDOUT and
 364   ACTUAL_STDOUT to match, every line in ACTUAL_STDOUT must match the
 365   EXPECTED_STDOUT regex, unless ALL_STDOUT is false.  For
 366   EXPECTED_STDERR regexes only one line in ACTUAL_STDERR need match."""
 367   expected_stderr = createExpectedOutput(expected_stderr, 'stderr', False)
 368   expected_stdout = createExpectedOutput(expected_stdout, 'stdout', all_stdout)
 369
 370   for (actual, expected, label, raisable) in (
 371       (actual_stderr, expected_stderr, 'STDERR', SVNExpectedStderr),
 372       (actual_stdout, expected_stdout, 'STDOUT', SVNExpectedStdout)):
 373     if expected is None:
 374       continue
 375
 376     if isinstance(expected, RegexOutput):
 377       raisable = svntest.main.SVNUnmatchedError
 378     elif not isinstance(expected, AnyOutput):
 379       raisable = svntest.main.SVNLineUnequal
 380
 381     compare_and_display_lines(message, label, expected, actual, raisable)
 382
 383 def verify_exit_code(message, actual, expected,
 384                      raisable=SVNUnexpectedExitCode):
 385   """Compare and display expected vs. actual exit codes:
 386   if they don't match, print the difference (preceded by MESSAGE iff
 387   not None) and raise an exception."""
 388
 389   if expected != actual:
 390     display_lines(message, "Exit Code",
 391                   str(expected) + '\n', str(actual) + '\n')
 392     raise raisable