framework/summary/common.py

   1 # Copyright 2013-2016 Intel Corporation
   2 # Copyright 2013, 2014 Advanced Micro Devices
   3 # Copyright 2014 VMWare
   4
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 # SOFTWARE.
  22
  23 """Shared functions for summary generation."""
  24
  25 from __future__ import (
  26     absolute_import, division, print_function, unicode_literals
  27 )
  28 import re
  29 import operator
  30
  31 import six
  32 from six.moves import zip
  33
  34 # a local variable status exists, prevent accidental overloading by renaming
  35 # the module
  36 import framework.status as so
  37 from framework.core import lazy_property
  38 from framework import grouptools
  39
  40
  41 class Results(object):  # pylint: disable=too-few-public-methods
  42     """Container object for results.
  43
  44     Has the results, the names of status, and the counts of statuses.
  45
  46     """
  47     def __init__(self, results):
  48         self.results = results
  49         self.names = Names(self)
  50         self.counts = Counts(self)
  51
  52     def get_result(self, name):
  53         """Get all results for a single test.
  54
  55         Replace any missing vaules with status.NOTRUN, correctly handles
  56         subtests.
  57
  58         """
  59         results = []
  60         for res in self.results:
  61             try:
  62                 results.append(res.get_result(name))
  63             except KeyError:
  64                 results.append(so.NOTRUN)
  65         return results
  66
  67
  68 class Names(object):
  69     """Class containing names of tests for various statuses.
  70
  71     Members contain lists of sets of names that have a status.
  72
  73     Each status is lazily evaluated and cached.
  74
  75     """
  76     def __init__(self, tests):
  77         self.__results = tests.results
  78
  79     def __diff(self, comparator, handler=None):
  80         """Helper for simplifying comparators using find_diffs."""
  81         ret = ['']
  82         if handler is None:
  83             ret.extend(find_diffs(self.__results, self.all, comparator))
  84         else:
  85             ret.extend(find_diffs(self.__results, self.all, comparator,
  86                                   handler=handler))
  87         return ret
  88
  89     def __single(self, comparator):
  90         """Helper for simplifying comparators using find_single."""
  91         return find_single(self.__results, self.all, comparator)
  92
  93     @lazy_property
  94     def all(self):
  95         """A set of all tests in all runs."""
  96         all_ = set()
  97         for res in self.__results:
  98             for key, value in six.iteritems(res.tests):
  99                 if not value.subtests:
 100                     all_.add(key)
 101                 else:
 102                     for subt in six.iterkeys(value.subtests):
 103                         all_.add(grouptools.join(key, subt))
 104         return all_
 105
 106     @lazy_property
 107     def changes(self):
 108         def handler(names, name, prev, cur):
 109             """Handle missing tests.
 110
 111             For changes we want literally anything where the first result
 112             isn't the same as the second result.
 113
 114             """
 115             def _get(res):
 116                 try:
 117                     return res.get_result(name)
 118                 except KeyError:
 119                     return so.NOTRUN
 120
 121             # Add any case of a != b except skip <-> notrun
 122             cur = _get(cur)
 123             prev = _get(prev)
 124             if cur != prev and {cur, prev} != {so.SKIP, so.NOTRUN}:
 125                 names.add(name)
 126
 127         return self.__diff(operator.ne, handler=handler)
 128
 129     @lazy_property
 130     def problems(self):
 131         return self.__single(lambda x: x > so.PASS)
 132
 133     @lazy_property
 134     def skips(self):
 135         # It is critical to use is not == here, otherwise so.NOTRUN will also
 136         # be added to this list
 137         return self.__single(lambda x: x is so.SKIP)
 138
 139     @lazy_property
 140     def regressions(self):
 141         # By ensureing tha min(x, y) is >= so.PASS we eleminate NOTRUN and SKIP
 142         # from these pages
 143         return self.__diff(lambda x, y: x < y and min(x, y) >= so.PASS)
 144
 145     @lazy_property
 146     def fixes(self):
 147         # By ensureing tha min(x, y) is >= so.PASS we eleminate NOTRUN and SKIP
 148         # from these pages
 149         return self.__diff(lambda x, y: x > y and min(x, y) >= so.PASS)
 150
 151     @lazy_property
 152     def enabled(self):
 153         def handler(names, name, prev, cur):
 154             if _result_in(name, cur) and not _result_in(name, prev):
 155                 names.add(name)
 156
 157         return self.__diff(
 158             lambda x, y: x is so.NOTRUN and y is not so.NOTRUN,
 159             handler=handler)
 160
 161     @lazy_property
 162     def disabled(self):
 163         def handler(names, name, prev, cur):
 164             if _result_in(name, prev) and not _result_in(name, cur):
 165                 names.add(name)
 166
 167         return self.__diff(
 168             lambda x, y: x is not so.NOTRUN and y is so.NOTRUN,
 169             handler=handler)
 170
 171     @lazy_property
 172     def incomplete(self):
 173         return self.__single(lambda x: x is so.INCOMPLETE)
 174
 175     @lazy_property
 176     def all_changes(self):
 177         if len(self.changes) > 1:
 178             return set.union(*self.changes[1:])
 179         else:
 180             return set()
 181
 182     @lazy_property
 183     def all_disabled(self):
 184         if len(self.disabled) > 1:
 185             return set.union(*self.disabled[1:])
 186         else:
 187             return set()
 188
 189     @lazy_property
 190     def all_enabled(self):
 191         if len(self.enabled) > 1:
 192             return set.union(*self.enabled[1:])
 193         else:
 194             return set()
 195
 196     @lazy_property
 197     def all_fixes(self):
 198         if len(self.fixes) > 1:
 199             return set.union(*self.fixes[1:])
 200         else:
 201             return set()
 202
 203     @lazy_property
 204     def all_regressions(self):
 205         if len(self.regressions) > 1:
 206             return set.union(*self.regressions[1:])
 207         else:
 208             return set()
 209
 210     @lazy_property
 211     def all_incomplete(self):
 212         if len(self.incomplete) > 1:
 213             return set.union(*self.incomplete)
 214         else:
 215             return self.incomplete[0]
 216
 217     @lazy_property
 218     def all_problems(self):
 219         if len(self.problems) > 1:
 220             return set.union(*self.problems)
 221         else:
 222             return self.problems[0]
 223
 224     @lazy_property
 225     def all_skips(self):
 226         if len(self.skips) > 1:
 227             return set.union(*self.skips)
 228         else:
 229             return self.skips[0]
 230
 231
 232 class Counts(object):
 233     """Number of tests in each category."""
 234     def __init__(self, tests):
 235         self.__names = tests.names
 236
 237     @lazy_property
 238     def all(self):
 239         return len(self.__names.all)
 240
 241     @lazy_property
 242     def changes(self):
 243         return [len(x) for x in self.__names.changes]
 244
 245     @lazy_property
 246     def problems(self):
 247         return [len(x) for x in self.__names.problems]
 248
 249     @lazy_property
 250     def skips(self):
 251         return [len(x) for x in self.__names.skips]
 252
 253     @lazy_property
 254     def regressions(self):
 255         return [len(x) for x in self.__names.regressions]
 256
 257     @lazy_property
 258     def fixes(self):
 259         return [len(x) for x in self.__names.fixes]
 260
 261     @lazy_property
 262     def enabled(self):
 263         return [len(x) for x in self.__names.enabled]
 264
 265     @lazy_property
 266     def disabled(self):
 267         return [len(x) for x in self.__names.disabled]
 268
 269     @lazy_property
 270     def incomplete(self):
 271         return [len(x) for x in self.__names.incomplete]
 272
 273
 274 def escape_filename(key):
 275     """Avoid reserved characters in filenames."""
 276     return re.sub(r'[<>:"|?*#]', '_', key)
 277
 278
 279 def escape_pathname(key):
 280     """ Remove / and \\ from names """
 281     return re.sub(r'[/\\]', '_', key)
 282
 283
 284 def _result_in(name, result):
 285     """If a result (or a subtest result) exists return True, else False."""
 286     try:
 287         # This is a little hacky, but I don't know of a better way where we
 288         # ensure the value is truthy
 289         _ = result.get_result(name)
 290         return True
 291     except KeyError:
 292         return False
 293
 294
 295 def find_diffs(results, tests, comparator, handler=lambda *a: None):
 296     """Generate diffs between two or more sets of results.
 297
 298     Arguments:
 299     results -- a list of results.TestrunResult instances
 300     tests -- an iterable of test names. Must be iterable more than once
 301     comparator -- a function with the signautre f(x, y), that returns True when
 302                   the test should be added to the set of diffs
 303
 304     Keyword Arguemnts:
 305     handler -- a function with the signature f(names, name, prev, cur). in the
 306                event of a KeyError while comparing the results with comparator,
 307                handler will be passed the (<the set of names>, <the current
 308                test name>, <the previous result>, <the current result>). This
 309                can be used to add name even when a KeyError is expected (ie,
 310                enabled tests).
 311                Default: pass
 312
 313     """
 314     diffs = [] # There can't be changes from nil -> 0
 315     for prev, cur in zip(results[:-1], results[1:]):
 316         names = set()
 317         for name in tests:
 318             try:
 319                 if comparator(prev.get_result(name), cur.get_result(name)):
 320                     names.add(name)
 321             except KeyError:
 322                 handler(names, name, prev, cur)
 323         diffs.append(names)
 324     return diffs
 325
 326
 327 def find_single(results, tests, func):
 328     """Find statuses in a single run."""
 329     statuses = []
 330     for res in results:
 331         names = set()
 332         for name in tests:
 333             try:
 334                 if func(res.get_result(name)):
 335                     names.add(name)
 336             except KeyError:
 337                 pass
 338         statuses.append(names)
 339     return statuses