framework/summary/common.py

   1 # coding=utf-8
   2 # Copyright 2013-2016, 2019 Intel Corporation
   3 # Copyright 2013, 2014 Advanced Micro Devices
   4 # Copyright 2014 VMWare
   5
   6 # Permission is hereby granted, free of charge, to any person obtaining a copy
   7 # of this software and associated documentation files (the "Software"), to deal
   8 # in the Software without restriction, including without limitation the rights
   9 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10 # copies of the Software, and to permit persons to whom the Software is
  11 # furnished to do so, subject to the following conditions:
  12
  13 # The above copyright notice and this permission notice shall be included in
  14 # all copies or substantial portions of the Software.
  15
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  19 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22 # SOFTWARE.
  23
  24 """Shared functions for summary generation."""
  25
  26 import re
  27 import operator
  28
  29 # a local variable status exists, prevent accidental overloading by renaming
  30 # the module
  31 import framework.status as so
  32 from framework.core import lazy_property
  33 from framework import grouptools
  34
  35
  36 class Results(object):  # pylint: disable=too-few-public-methods
  37     """Container object for results.
  38
  39     Has the results, the names of status, and the counts of statuses.
  40
  41     """
  42     def __init__(self, results):
  43         self.results = results
  44         self.names = Names(self)
  45         self.counts = Counts(self)
  46
  47     def get_result(self, name):
  48         """Get all results for a single test.
  49
  50         Replace any missing values with status.NOTRUN, correctly handles
  51         subtests.
  52
  53         """
  54         results = []
  55         for res in self.results:
  56             try:
  57                 results.append(res.get_result(name))
  58             except KeyError:
  59                 results.append(so.NOTRUN)
  60         return results
  61
  62
  63 class Names(object):
  64     """Class containing names of tests for various statuses.
  65
  66     Members contain lists of sets of names that have a status.
  67
  68     Each status is lazily evaluated and cached.
  69
  70     """
  71     def __init__(self, tests):
  72         self.__results = tests.results
  73
  74     def __diff(self, comparator, handler=None):
  75         """Helper for simplifying comparators using find_diffs."""
  76         ret = ['']
  77         if handler is None:
  78             ret.extend(find_diffs(self.__results, self.all, comparator))
  79         else:
  80             ret.extend(find_diffs(self.__results, self.all, comparator,
  81                                   handler=handler))
  82         return ret
  83
  84     def __single(self, comparator):
  85         """Helper for simplifying comparators using find_single."""
  86         return find_single(self.__results, self.all, comparator)
  87
  88     @lazy_property
  89     def all(self):
  90         """A set of all tests in all runs."""
  91         all_ = set()
  92         for res in self.__results:
  93             for key, value in res.tests.items():
  94                 if not value.subtests:
  95                     all_.add(key)
  96                 else:
  97                     for subt in value.subtests.keys():
  98                         all_.add(grouptools.join(key, subt))
  99         return all_
 100
 101     @lazy_property
 102     def changes(self):
 103         def handler(names, name, prev, cur):
 104             """Handle missing tests.
 105
 106             For changes we want literally anything where the first result
 107             isn't the same as the second result.
 108
 109             """
 110             def _get(res):
 111                 try:
 112                     return res.get_result(name)
 113                 except KeyError:
 114                     return so.NOTRUN
 115
 116             # Add any case of a != b except skip <-> notrun
 117             cur = _get(cur)
 118             prev = _get(prev)
 119             if cur != prev and {cur, prev} != {so.SKIP, so.NOTRUN}:
 120                 names.add(name)
 121
 122         return self.__diff(operator.ne, handler=handler)
 123
 124     @lazy_property
 125     def problems(self):
 126         return self.__single(lambda x: x > so.PASS)
 127
 128     @lazy_property
 129     def skips(self):
 130         # It is critical to use is not == here, otherwise so.NOTRUN will also
 131         # be added to this list
 132         return self.__single(lambda x: x is so.SKIP)
 133
 134     @lazy_property
 135     def regressions(self):
 136         # By ensureing that min(x, y) is >= so.PASS we eliminate NOTRUN and SKIP
 137         # from these pages
 138         return self.__diff(lambda x, y: x < y and min(x, y) >= so.PASS)
 139
 140     @lazy_property
 141     def fixes(self):
 142         # By ensureing that min(x, y) is >= so.PASS we eliminate NOTRUN and SKIP
 143         # from these pages
 144         return self.__diff(lambda x, y: x > y and min(x, y) >= so.PASS)
 145
 146     @lazy_property
 147     def enabled(self):
 148         def handler(names, name, prev, cur):
 149             if _result_in(name, cur) and not _result_in(name, prev):
 150                 names.add(name)
 151
 152         return self.__diff(
 153             lambda x, y: x is so.NOTRUN and y is not so.NOTRUN,
 154             handler=handler)
 155
 156     @lazy_property
 157     def disabled(self):
 158         def handler(names, name, prev, cur):
 159             if _result_in(name, prev) and not _result_in(name, cur):
 160                 names.add(name)
 161
 162         return self.__diff(
 163             lambda x, y: x is not so.NOTRUN and y is so.NOTRUN,
 164             handler=handler)
 165
 166     @lazy_property
 167     def incomplete(self):
 168         return self.__single(lambda x: x is so.INCOMPLETE)
 169
 170     @lazy_property
 171     def all_changes(self):
 172         if len(self.changes) > 1:
 173             return set.union(*self.changes[1:])
 174         else:
 175             return set()
 176
 177     @lazy_property
 178     def all_disabled(self):
 179         if len(self.disabled) > 1:
 180             return set.union(*self.disabled[1:])
 181         else:
 182             return set()
 183
 184     @lazy_property
 185     def all_enabled(self):
 186         if len(self.enabled) > 1:
 187             return set.union(*self.enabled[1:])
 188         else:
 189             return set()
 190
 191     @lazy_property
 192     def all_fixes(self):
 193         if len(self.fixes) > 1:
 194             return set.union(*self.fixes[1:])
 195         else:
 196             return set()
 197
 198     @lazy_property
 199     def all_regressions(self):
 200         if len(self.regressions) > 1:
 201             return set.union(*self.regressions[1:])
 202         else:
 203             return set()
 204
 205     @lazy_property
 206     def all_incomplete(self):
 207         if len(self.incomplete) > 1:
 208             return set.union(*self.incomplete)
 209         else:
 210             return self.incomplete[0]
 211
 212     @lazy_property
 213     def all_problems(self):
 214         if len(self.problems) > 1:
 215             return set.union(*self.problems)
 216         else:
 217             return self.problems[0]
 218
 219     @lazy_property
 220     def all_skips(self):
 221         if len(self.skips) > 1:
 222             return set.union(*self.skips)
 223         else:
 224             return self.skips[0]
 225
 226
 227 class Counts(object):
 228     """Number of tests in each category."""
 229     def __init__(self, tests):
 230         self.__names = tests.names
 231
 232     @lazy_property
 233     def all(self):
 234         return len(self.__names.all)
 235
 236     @lazy_property
 237     def changes(self):
 238         return [len(x) for x in self.__names.changes]
 239
 240     @lazy_property
 241     def problems(self):
 242         return [len(x) for x in self.__names.problems]
 243
 244     @lazy_property
 245     def skips(self):
 246         return [len(x) for x in self.__names.skips]
 247
 248     @lazy_property
 249     def regressions(self):
 250         return [len(x) for x in self.__names.regressions]
 251
 252     @lazy_property
 253     def fixes(self):
 254         return [len(x) for x in self.__names.fixes]
 255
 256     @lazy_property
 257     def enabled(self):
 258         return [len(x) for x in self.__names.enabled]
 259
 260     @lazy_property
 261     def disabled(self):
 262         return [len(x) for x in self.__names.disabled]
 263
 264     @lazy_property
 265     def incomplete(self):
 266         return [len(x) for x in self.__names.incomplete]
 267
 268
 269 def escape_filename(key):
 270     """Avoid reserved characters in filenames."""
 271     return re.sub(r'[<>:"|?*#]', '_', key)
 272
 273
 274 def escape_pathname(key):
 275     """ Remove / and \\ from names """
 276     return re.sub(r'[/\\]', '_', key)
 277
 278
 279 def _result_in(name, result):
 280     """If a result (or a subtest result) exists return True, else False."""
 281     try:
 282         # This is a little hacky, but I don't know of a better way where we
 283         # ensure the value is truthy
 284         _ = result.get_result(name)
 285         return True
 286     except KeyError:
 287         return False
 288
 289
 290 def find_diffs(results, tests, comparator, handler=lambda *a: None):
 291     """Generate diffs between two or more sets of results.
 292
 293     Arguments:
 294     results -- a list of results.TestrunResult instances
 295     tests -- an iterable of test names. Must be iterable more than once
 296     comparator -- a function with the signautre f(x, y), that returns True when
 297                   the test should be added to the set of diffs
 298
 299     Keyword Arguments:
 300     handler -- a function with the signature f(names, name, prev, cur). in the
 301                event of a KeyError while comparing the results with comparator,
 302                handler will be passed the (<the set of names>, <the current
 303                test name>, <the previous result>, <the current result>). This
 304                can be used to add name even when a KeyError is expected (ie,
 305                enabled tests).
 306                Default: pass
 307
 308     """
 309     diffs = [] # There can't be changes from nil -> 0
 310     for prev, cur in zip(results[:-1], results[1:]):
 311         names = set()
 312         for name in tests:
 313             try:
 314                 if comparator(prev.get_result(name), cur.get_result(name)):
 315                     names.add(name)
 316             except KeyError:
 317                 handler(names, name, prev, cur)
 318         diffs.append(names)
 319     return diffs
 320
 321
 322 def find_single(results, tests, func):
 323     """Find statuses in a single run."""
 324     statuses = []
 325     for res in results:
 326         names = set()
 327         for name in tests:
 328             try:
 329                 if func(res.get_result(name)):
 330                     names.add(name)
 331             except KeyError:
 332                 pass
 333         statuses.append(names)
 334     return statuses