framework/summary.py

   1 #!/usr/bin/env python
   2 #
   3 # Permission is hereby granted, free of charge, to any person
   4 # obtaining a copy of this software and associated documentation
   5 # files (the "Software"), to deal in the Software without
   6 # restriction, including without limitation the rights to use,
   7 # copy, modify, merge, publish, distribute, sublicense, and/or
   8 # sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following
  10 # conditions:
  11 #
  12 # This permission notice shall be included in all copies or
  13 # substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  18 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  20 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  21 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22 # DEALINGS IN THE SOFTWARE.
  23
  24 import core
  25
  26
  27 #############################################################################
  28 ##### Vector indicating the number of subtests that have passed/failed/etc.
  29 #############################################################################
  30 class PassVector:
  31         def __init__(self, p, w, f, s):
  32                 self.passnr = p
  33                 self.warnnr = w
  34                 self.failnr = f
  35                 self.skipnr = s
  36
  37         def add(self, o):
  38                 self.passnr += o.passnr
  39                 self.warnnr += o.warnnr
  40                 self.failnr += o.failnr
  41                 self.skipnr += o.skipnr
  42
  43
  44 #############################################################################
  45 ##### TestSummary: Summarize the results for one test across a
  46 ##### number of testruns
  47 #############################################################################
  48 class TestSummary:
  49         def __init__(self, summary, path, name, results):
  50                 """\
  51 summary is the root summary object
  52 path is the path to the group (e.g. shaders/glean-fragProg1)
  53 name is the display name of the group (e.g. glean-fragProg1)
  54 results is an array of TestResult instances, one per testrun
  55 """
  56                 self.summary = summary
  57                 self.path = path
  58                 self.name = name
  59                 self.results = results[:]
  60
  61                 for j in range(len(self.results)):
  62                         result = self.results[j]
  63                         result.testrun = self.summary.testruns[j]
  64                         result.status = ''
  65                         if 'result' in result:
  66                                 result.status = result['result']
  67
  68                         vectormap = {
  69                                 'pass': PassVector(1,0,0,0),
  70                                 'warn': PassVector(0,1,0,0),
  71                                 'fail': PassVector(0,0,1,0),
  72                                 'skip': PassVector(0,0,0,1)
  73                         }
  74
  75                         if result.status not in vectormap:
  76                                 result.status = 'warn'
  77
  78                         result.passvector = vectormap[result.status]
  79
  80                 stati = set([result.status for result in results])
  81                 self.changes = len(stati) > 1
  82                 self.problems = len(stati - set(['pass', 'skip'])) > 0
  83
  84         def allTests(self):
  85                 return [self]
  86
  87 #############################################################################
  88 ##### GroupSummary: Summarize a group of tests
  89 #############################################################################
  90 class GroupSummary:
  91         def __init__(self, summary, path, name, results):
  92                 """\
  93 summary is the root summary object
  94 path is the path to the group (e.g. shaders/glean-fragProg1)
  95 name is the display name of the group (e.g. glean-fragProg1)
  96 results is an array of GroupResult instances, one per testrun
  97 """
  98                 self.summary = summary
  99                 self.path = path
 100                 self.name = name
 101                 self.results = results[:]
 102                 self.changes = False
 103                 self.problems = False
 104                 self.children = {}
 105
 106                 # Perform some initial annotations
 107                 for j in range(len(self.results)):
 108                         result = self.results[j]
 109                         result.passvector = PassVector(0, 0, 0, 0)
 110                         result.testrun = self.summary.testruns[j]
 111
 112                 # Collect, create and annotate children
 113                 for result in self.results:
 114                         for name in result:
 115                                 if name in self.children:
 116                                         continue
 117
 118                                 childpath = name
 119                                 if len(self.path) > 0:
 120                                         childpath = self.path + '/' + childpath
 121
 122                                 if isinstance(result[name], core.GroupResult):
 123                                         childresults = [r.get(name, core.GroupResult())
 124                                                         for r in self.results]
 125
 126                                         self.children[name] = GroupSummary(
 127                                                 summary,
 128                                                 childpath,
 129                                                 name,
 130                                                 childresults
 131                                         )
 132                                 else:
 133                                         childresults = [r.get(name, core.TestResult({}, { 'result': 'skip' }))
 134                                                         for r in self.results]
 135
 136                                         self.children[name] = TestSummary(
 137                                                 summary,
 138                                                 childpath,
 139                                                 name,
 140                                                 childresults
 141                                         )
 142
 143                                 for j in range(len(self.results)):
 144                                         self.results[j].passvector.add(childresults[j].passvector)
 145
 146                                 self.changes = self.changes or self.children[name].changes
 147                                 self.problems = self.problems or self.children[name].problems
 148
 149         def allTests(self):
 150                 """\
 151 Returns an array of all child TestSummary instances.
 152 """
 153                 return [t for name in self.children for t in self.children[name].allTests()]
 154
 155 #############################################################################
 156 ##### Summary: Summarize an array of testruns
 157 #############################################################################
 158 class Summary:
 159         def __init__(self, testruns):
 160                 """\
 161 testruns is an array of TestrunResult instances
 162 """
 163                 self.testruns = testruns
 164                 self.root = GroupSummary(self, '', 'All', [tr.results for tr in testruns])
 165
 166         def allTests(self):
 167                 """\
 168 Returns an array of all child TestSummary instances.
 169 """
 170                 return self.root.allTests()