framework/programs/summary.py

   1 # coding=utf-8
   2 # Permission is hereby granted, free of charge, to any person
   3 # obtaining a copy of this software and associated documentation
   4 # files (the "Software"), to deal in the Software without
   5 # restriction, including without limitation the rights to use,
   6 # copy, modify, merge, publish, distribute, sublicense, and/or
   7 # sell copies of the Software, and to permit persons to whom the
   8 # Software is furnished to do so, subject to the following
   9 # conditions:
  10 #
  11 # This permission notice shall be included in all copies or
  12 # substantial portions of the Software.
  13 #
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  15 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  16 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  17 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  18 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  19 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  20 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21 # DEALINGS IN THE SOFTWARE.
  22
  23 from __future__ import (
  24     absolute_import, division, print_function, unicode_literals
  25 )
  26 import argparse
  27 import shutil
  28 import os
  29 import os.path as path
  30 import sys
  31 import errno
  32
  33 import six
  34
  35 from framework import summary, status, core, backends, exceptions
  36 from . import parsers
  37
  38 __all__ = [
  39     'aggregate',
  40     'console',
  41     'csv',
  42     'html',
  43     'feature'
  44     'formatted'
  45 ]
  46
  47 DEFAULT_FMT_STR="{name} ::: {time} ::: {returncode} ::: {result}"
  48
  49 @exceptions.handler
  50 def html(input_):
  51     # Make a copy of the status text list and add all. This is used as the
  52     # argument list for -e/--exclude
  53     statuses = set(str(s) for s in status.ALL)
  54     statuses.add('all')
  55
  56     """Combine files in a tests/ directory into a single results file."""
  57     unparsed = parsers.parse_config(input_)[1]
  58
  59     # Adding the parent is necissary to get the help options
  60     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
  61     parser.add_argument("-o", "--overwrite",
  62                         action="store_true",
  63                         help="Overwrite existing directories")
  64     parser.add_argument("-l", "--list",
  65                         action="store",
  66                         help="Load a newline separated list of results. These "
  67                              "results will be prepended to any Results "
  68                              "specified on the command line")
  69     parser.add_argument("-e", "--exclude-details",
  70                         default=[],
  71                         action="append",
  72                         choices=statuses,
  73                         help="Optionally exclude the generation of HTML pages "
  74                              "for individual test pages with the status(es) "
  75                              "given as arguments. This speeds up HTML "
  76                              "generation, but reduces the info in the HTML "
  77                              "pages. May be used multiple times")
  78     parser.add_argument("summaryDir",
  79                         metavar="<Summary Directory>",
  80                         help="Directory to put HTML files in")
  81     parser.add_argument("resultsFiles",
  82                         metavar="<Results Files>",
  83                         nargs="*",
  84                         help="Results files to include in HTML")
  85     args = parser.parse_args(unparsed)
  86
  87     # If args.list and args.resultsFiles are empty, then raise an error
  88     if not args.list and not args.resultsFiles:
  89         raise parser.error("Missing required option -l or <resultsFiles>")
  90
  91     # Convert the exclude_details list to status objects, without this using
  92     # the -e option will except
  93     if args.exclude_details:
  94         # If exclude-results has all, then change it to be all
  95         if 'all' in args.exclude_details:
  96             args.exclude_details = status.ALL
  97         else:
  98             args.exclude_details = frozenset(
  99                 status.status_lookup(i) for i in args.exclude_details)
 100
 101
 102     # if overwrite is requested delete the output directory
 103     if path.exists(args.summaryDir) and args.overwrite:
 104         shutil.rmtree(args.summaryDir)
 105
 106     # If the requested directory doesn't exist, create it or throw an error
 107     try:
 108         core.check_dir(args.summaryDir, not args.overwrite)
 109     except exceptions.PiglitException:
 110         raise exceptions.PiglitFatalError(
 111             '{} already exists.\n'
 112             'use -o/--overwrite if you want to overwrite it.'.format(
 113                 args.summaryDir))
 114
 115     # Merge args.list and args.resultsFiles
 116     if args.list:
 117         args.resultsFiles.extend(core.parse_listfile(args.list))
 118
 119     # Create the HTML output
 120     summary.html(args.resultsFiles, args.summaryDir, args.exclude_details)
 121
 122
 123 @exceptions.handler
 124 def console(input_):
 125     """Combine files in a tests/ directory into a single results file."""
 126     unparsed = parsers.parse_config(input_)[1]
 127
 128     # Adding the parent is necessary to get the help options
 129     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
 130
 131     # Set the -d and -s options as exclusive, since it's silly to call for diff
 132     # and then call for only summary
 133     excGroup1 = parser.add_mutually_exclusive_group()
 134     excGroup1.add_argument("-d", "--diff",
 135                            action="store_const",
 136                            const="diff",
 137                            dest='mode',
 138                            help="Only display the differences between multiple "
 139                                 "result files")
 140     excGroup1.add_argument("-s", "--summary",
 141                            action="store_const",
 142                            const="summary",
 143                            dest='mode',
 144                            help="Only display the summary, not the individual "
 145                                 "test results")
 146     excGroup1.add_argument("-i", "--incomplete",
 147                            action="store_const",
 148                            const="incomplete",
 149                            dest='mode',
 150                            help="Only display tests that are incomplete.")
 151     excGroup1.add_argument("-p", "--problems",
 152                            action="store_const",
 153                            const="problems",
 154                            dest='mode',
 155                            help="Only display tests that had problems.")
 156     parser.add_argument("-l", "--list",
 157                         action="store",
 158                         help="Use test results from a list file")
 159     parser.add_argument("results",
 160                         metavar="<Results Path(s)>",
 161                         nargs="+",
 162                         help="Space separated paths to at least one results "
 163                              "file")
 164     args = parser.parse_args(unparsed)
 165
 166     # Throw an error if -d/--diff is called, but only one results file is
 167     # provided
 168     if args.mode == 'diff' and len(args.results) < 2:
 169         parser.error('-d/--diff cannot be specified unless two or more '
 170                      'results files are specified')
 171
 172     # make list of results
 173     if args.list:
 174         args.results.extend(core.parse_listfile(args.list))
 175
 176     # Generate the output
 177     summary.console(args.results, args.mode or 'all')
 178
 179
 180 @exceptions.handler
 181 def csv(input_):
 182     format_string="{name},{time},{returncode},{result}"
 183     return formatted(input_, default_format_string=format_string)
 184
 185 @exceptions.handler
 186 def formatted(input_, default_format_string=DEFAULT_FMT_STR):
 187     # Make a copy of the status text list and add all. This is used as the
 188     # argument list for -e/--exclude
 189     statuses = set(str(s) for s in status.ALL)
 190
 191     unparsed = parsers.parse_config(input_)[1]
 192
 193     # Adding the parent is necissary to get the help options
 194     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
 195     parser.add_argument("--format",
 196                         dest="format_string",
 197                         metavar="<format string>",
 198                         default=default_format_string,
 199                         action="store",
 200                         help="A template string that defines the format. "
 201                              "Replacement tokens are {name}, {time}, "
 202                              "{returncode} and {result}")
 203     parser.add_argument("-e", "--exclude-details",
 204                         default=[],
 205                         action="append",
 206                         choices=statuses,
 207                         help="Optionally exclude the listing of tests with "
 208                              "the status(es) given as arguments. "
 209                              "May be used multiple times")
 210     parser.add_argument("-o", "--output",
 211                         metavar="<Output File>",
 212                         action="store",
 213                         dest="output",
 214                         default="stdout",
 215                         help="Output filename")
 216     parser.add_argument("test_results",
 217                         metavar="<Input Files>",
 218                         help="JSON results file to be converted")
 219     args = parser.parse_args(unparsed)
 220
 221     testrun = backends.load(args.test_results)
 222
 223     def write_results(output):
 224         for name, result in six.iteritems(testrun.tests):
 225             if result.result in args.exclude_details:
 226                 continue
 227             output.write((args.format_string + "\n").format(
 228                 name=name,
 229                 time=result.time.total,
 230                 returncode=result.returncode,
 231                 result=result.result))
 232
 233     if args.output != "stdout":
 234         with open(args.output, 'w') as output:
 235             write_results(output)
 236     else:
 237         write_results(sys.stdout)
 238
 239
 240 @exceptions.handler
 241 def aggregate(input_):
 242     """Combine files in a tests/ directory into a single results file."""
 243     unparsed = parsers.parse_config(input_)[1]
 244
 245     # Adding the parent is necissary to get the help options
 246     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
 247     parser.add_argument('results_folder',
 248                         type=path.realpath,
 249                         metavar="<results path>",
 250                         help="Path to a results directory "
 251                              "(which contains a tests directory)")
 252     parser.add_argument('-o', '--output',
 253                         default="results.json",
 254                         help="name of output file. Default: results.json")
 255     args = parser.parse_args(unparsed)
 256
 257     assert os.path.isdir(args.results_folder)
 258
 259     # args.results_folder must be a path with a 'tests' directory in it, not
 260     # the tests directory itself.
 261     outfile = os.path.join(args.results_folder, args.output)
 262     try:
 263         results = backends.load(args.results_folder)
 264     except backends.BackendError:
 265         raise exceptions.PiglitFatalError(
 266             'Cannot find a tests directory to aggregate in {}.\n'
 267             'Are you you sure that you pointed to '
 268             'a results directory (not results/tests)?'.format(args.results_folder))
 269
 270     try:
 271         # FIXME: This works, it fixes the problem, but it only works because
 272         # only the json backend has the ability to aggregate results at the
 273         # moment.
 274         backends.json._write(results, outfile)
 275     except IOError as e:
 276         if e.errno == errno.EPERM:
 277             raise exceptions.PiglitFatalError(
 278                 "Unable to write aggregated file, permission denied.")
 279         raise
 280
 281     print("Aggregated file written to: {}.{}".format(
 282         outfile, backends.compression.get_mode()))
 283
 284
 285 @exceptions.handler
 286 def feature(input_):
 287     parser = argparse.ArgumentParser()
 288     parser.add_argument("-o", "--overwrite",
 289                         action="store_true",
 290                         help="Overwrite existing directories")
 291     parser.add_argument("featureFile",
 292                         metavar="<Feature json file>",
 293                         help="Json file containing the features description")
 294     parser.add_argument("summaryDir",
 295                         metavar="<Summary Directory>",
 296                         help="Directory to put HTML files in")
 297     parser.add_argument("resultsFiles",
 298                         metavar="<Results Files>",
 299                         nargs="*",
 300                         help="Results files to include in HTML")
 301     args = parser.parse_args(input_)
 302
 303     # If args.list and args.resultsFiles are empty, then raise an error
 304     if not args.featureFile and not args.resultsFiles:
 305         raise parser.error("Missing required option -l or <resultsFiles>")
 306
 307     # If args.list and args.resultsFiles are empty, then raise an error
 308     if not args.resultsFiles or not path.exists(args.featureFile):
 309         raise parser.error("Missing json file")
 310
 311     # if overwrite is requested delete the output directory
 312     if path.exists(args.summaryDir) and args.overwrite:
 313         shutil.rmtree(args.summaryDir)
 314
 315     # If the requested directory doesn't exist, create it or throw an error
 316     try:
 317         core.check_dir(args.summaryDir, not args.overwrite)
 318     except exceptions.PiglitException:
 319         raise exceptions.PiglitFatalError(
 320             '{} already exists.\n'
 321             'use -o/--overwrite if you want to overwrite it.'.format(
 322                 args.summaryDir))
 323
 324     summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)