framework/programs/summary.py

   1 # coding=utf-8
   2 # Permission is hereby granted, free of charge, to any person
   3 # obtaining a copy of this software and associated documentation
   4 # files (the "Software"), to deal in the Software without
   5 # restriction, including without limitation the rights to use,
   6 # copy, modify, merge, publish, distribute, sublicense, and/or
   7 # sell copies of the Software, and to permit persons to whom the
   8 # Software is furnished to do so, subject to the following
   9 # conditions:
  10 #
  11 # This permission notice shall be included in all copies or
  12 # substantial portions of the Software.
  13 #
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  15 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  16 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  17 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  18 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  19 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  20 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21 # DEALINGS IN THE SOFTWARE.
  22
  23 import argparse
  24 import shutil
  25 import os
  26 import os.path as path
  27 import sys
  28 import errno
  29
  30 from framework import summary, status, core, backends, exceptions
  31 from . import parsers
  32
  33 __all__ = [
  34     'aggregate',
  35     'console',
  36     'csv',
  37     'html',
  38     'feature'
  39     'formatted'
  40 ]
  41
  42 DEFAULT_FMT_STR="{name} ::: {time} ::: {returncode} ::: {result}"
  43
  44 @exceptions.handler
  45 def html(input_):
  46     # Make a copy of the status text list and add all. This is used as the
  47     # argument list for -e/--exclude
  48     statuses = set(str(s) for s in status.ALL)
  49     statuses.add('all')
  50
  51     """Combine files in a tests/ directory into a single results file."""
  52     unparsed = parsers.parse_config(input_)[1]
  53
  54     # Adding the parent is necissary to get the help options
  55     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
  56     parser.add_argument("-o", "--overwrite",
  57                         action="store_true",
  58                         help="Overwrite existing directories")
  59     parser.add_argument("-l", "--list",
  60                         action="store",
  61                         help="Load a newline separated list of results. These "
  62                              "results will be prepended to any Results "
  63                              "specified on the command line")
  64     parser.add_argument("-e", "--exclude-details",
  65                         default=[],
  66                         action="append",
  67                         choices=statuses,
  68                         help="Optionally exclude the generation of HTML pages "
  69                              "for individual test pages with the status(es) "
  70                              "given as arguments. This speeds up HTML "
  71                              "generation, but reduces the info in the HTML "
  72                              "pages. May be used multiple times")
  73     parser.add_argument("summaryDir",
  74                         metavar="<Summary Directory>",
  75                         help="Directory to put HTML files in")
  76     parser.add_argument("resultsFiles",
  77                         metavar="<Results Files>",
  78                         nargs="*",
  79                         help="Results files to include in HTML")
  80     args = parser.parse_args(unparsed)
  81
  82     # If args.list and args.resultsFiles are empty, then raise an error
  83     if not args.list and not args.resultsFiles:
  84         raise parser.error("Missing required option -l or <resultsFiles>")
  85
  86     # Convert the exclude_details list to status objects, without this using
  87     # the -e option will except
  88     if args.exclude_details:
  89         # If exclude-results has all, then change it to be all
  90         if 'all' in args.exclude_details:
  91             args.exclude_details = status.ALL
  92         else:
  93             args.exclude_details = frozenset(
  94                 status.status_lookup(i) for i in args.exclude_details)
  95
  96
  97     # if overwrite is requested delete the output directory
  98     if path.exists(args.summaryDir) and args.overwrite:
  99         shutil.rmtree(args.summaryDir)
 100
 101     # If the requested directory doesn't exist, create it or throw an error
 102     try:
 103         core.check_dir(args.summaryDir, not args.overwrite)
 104     except exceptions.PiglitException:
 105         raise exceptions.PiglitFatalError(
 106             '{} already exists.\n'
 107             'use -o/--overwrite if you want to overwrite it.'.format(
 108                 args.summaryDir))
 109
 110     # Merge args.list and args.resultsFiles
 111     if args.list:
 112         args.resultsFiles.extend(core.parse_listfile(args.list))
 113
 114     # Create the HTML output
 115     summary.html(args.resultsFiles, args.summaryDir, args.exclude_details)
 116
 117
 118 @exceptions.handler
 119 def console(input_):
 120     """Combine files in a tests/ directory into a single results file."""
 121     unparsed = parsers.parse_config(input_)[1]
 122
 123     # Adding the parent is necessary to get the help options
 124     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
 125
 126     # Set the -d and -s options as exclusive, since it's silly to call for diff
 127     # and then call for only summary
 128     excGroup1 = parser.add_mutually_exclusive_group()
 129     excGroup1.add_argument("-d", "--diff",
 130                            action="store_const",
 131                            const="diff",
 132                            dest='mode',
 133                            help="Only display the differences between multiple "
 134                                 "result files")
 135     excGroup1.add_argument("-s", "--summary",
 136                            action="store_const",
 137                            const="summary",
 138                            dest='mode',
 139                            help="Only display the summary, not the individual "
 140                                 "test results")
 141     excGroup1.add_argument("-g", "--fixes",
 142                            action="store_const",
 143                            const="fixes",
 144                            dest='mode',
 145                            help="Only display tests that have been fixed.")
 146     excGroup1.add_argument("-i", "--incomplete",
 147                            action="store_const",
 148                            const="incomplete",
 149                            dest='mode',
 150                            help="Only display tests that are incomplete.")
 151     excGroup1.add_argument("-p", "--problems",
 152                            action="store_const",
 153                            const="problems",
 154                            dest='mode',
 155                            help="Only display tests that had problems.")
 156     excGroup1.add_argument("-r", "--regressions",
 157                            action="store_const",
 158                            const="regressions",
 159                            dest='mode',
 160                            help="Only display tests that regressed.")
 161     parser.add_argument("-l", "--list",
 162                         action="store",
 163                         help="Use test results from a list file")
 164     parser.add_argument("results",
 165                         metavar="<Results Path(s)>",
 166                         nargs="+",
 167                         help="Space separated paths to at least one results "
 168                              "file")
 169     args = parser.parse_args(unparsed)
 170
 171     # Throw an error if -d/--diff is called, but only one results file is
 172     # provided
 173     if args.mode == 'diff' and len(args.results) < 2:
 174         parser.error('-d/--diff cannot be specified unless two or more '
 175                      'results files are specified')
 176
 177     # make list of results
 178     if args.list:
 179         args.results.extend(core.parse_listfile(args.list))
 180
 181     # Generate the output
 182     summary.console(args.results, args.mode or 'all')
 183
 184
 185 @exceptions.handler
 186 def csv(input_):
 187     format_string="{name},{time},{returncode},{result}"
 188     return formatted(input_, default_format_string=format_string)
 189
 190 @exceptions.handler
 191 def formatted(input_, default_format_string=DEFAULT_FMT_STR):
 192     # Make a copy of the status text list and add all. This is used as the
 193     # argument list for -e/--exclude
 194     statuses = set(str(s) for s in status.ALL)
 195
 196     unparsed = parsers.parse_config(input_)[1]
 197
 198     # Adding the parent is necissary to get the help options
 199     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
 200     parser.add_argument("--format",
 201                         dest="format_string",
 202                         metavar="<format string>",
 203                         default=default_format_string,
 204                         action="store",
 205                         help="A template string that defines the format. "
 206                              "Replacement tokens are {name}, {time}, "
 207                              "{returncode} and {result}")
 208     parser.add_argument("-e", "--exclude-details",
 209                         default=[],
 210                         action="append",
 211                         choices=statuses,
 212                         help="Optionally exclude the listing of tests with "
 213                              "the status(es) given as arguments. "
 214                              "May be used multiple times")
 215     parser.add_argument("-o", "--output",
 216                         metavar="<Output File>",
 217                         action="store",
 218                         dest="output",
 219                         default="stdout",
 220                         help="Output filename")
 221     parser.add_argument("test_results",
 222                         metavar="<Input Files>",
 223                         help="JSON results file to be converted")
 224     args = parser.parse_args(unparsed)
 225
 226     testrun = backends.load(args.test_results)
 227
 228     def write_results(output):
 229         for name, result in testrun.tests.items():
 230             if result.result in args.exclude_details:
 231                 continue
 232             output.write((args.format_string + "\n").format(
 233                 name=name,
 234                 time=result.time.total,
 235                 returncode=result.returncode,
 236                 result=result.result))
 237
 238     if args.output != "stdout":
 239         with open(args.output, 'w') as output:
 240             write_results(output)
 241     else:
 242         write_results(sys.stdout)
 243
 244
 245 @exceptions.handler
 246 def aggregate(input_):
 247     """Combine files in a tests/ directory into a single results file."""
 248     unparsed = parsers.parse_config(input_)[1]
 249
 250     # Adding the parent is necissary to get the help options
 251     parser = argparse.ArgumentParser(parents=[parsers.CONFIG])
 252     parser.add_argument('results_folder',
 253                         type=path.realpath,
 254                         metavar="<results path>",
 255                         help="Path to a results directory "
 256                              "(which contains a tests directory)")
 257     parser.add_argument('-o', '--output',
 258                         default="results.json",
 259                         help="name of output file. Default: results.json")
 260     args = parser.parse_args(unparsed)
 261
 262     assert os.path.isdir(args.results_folder)
 263
 264     # args.results_folder must be a path with a 'tests' directory in it, not
 265     # the tests directory itself.
 266     outfile = os.path.join(args.results_folder, args.output)
 267     try:
 268         results = backends.load(args.results_folder)
 269     except backends.BackendError:
 270         raise exceptions.PiglitFatalError(
 271             'Cannot find a tests directory to aggregate in {}.\n'
 272             'Are you you sure that you pointed to '
 273             'a results directory (not results/tests)?'.format(args.results_folder))
 274
 275     try:
 276         use_compression = backends.write(results, outfile)
 277     except IOError as e:
 278         if e.errno == errno.EPERM:
 279             raise exceptions.PiglitFatalError(
 280                 "Unable to write aggregated file, permission denied.")
 281         raise
 282
 283     mode = backends.compression.get_mode() if use_compression else 'none'
 284     comp_ext = '.{}'.format(mode) if mode != 'none' else ''
 285     print("Aggregated file written to: {}{}".format(outfile, comp_ext))
 286
 287
 288 @exceptions.handler
 289 def feature(input_):
 290     parser = argparse.ArgumentParser()
 291     parser.add_argument("-o", "--overwrite",
 292                         action="store_true",
 293                         help="Overwrite existing directories")
 294     parser.add_argument("featureFile",
 295                         metavar="<Feature json file>",
 296                         help="Json file containing the features description")
 297     parser.add_argument("summaryDir",
 298                         metavar="<Summary Directory>",
 299                         help="Directory to put HTML files in")
 300     parser.add_argument("resultsFiles",
 301                         metavar="<Results Files>",
 302                         nargs="*",
 303                         help="Results files to include in HTML")
 304     args = parser.parse_args(input_)
 305
 306     # If args.list and args.resultsFiles are empty, then raise an error
 307     if not args.featureFile and not args.resultsFiles:
 308         raise parser.error("Missing required option -l or <resultsFiles>")
 309
 310     # If args.list and args.resultsFiles are empty, then raise an error
 311     if not args.resultsFiles or not path.exists(args.featureFile):
 312         raise parser.error("Missing json file")
 313
 314     # if overwrite is requested delete the output directory
 315     if path.exists(args.summaryDir) and args.overwrite:
 316         shutil.rmtree(args.summaryDir)
 317
 318     # If the requested directory doesn't exist, create it or throw an error
 319     try:
 320         core.check_dir(args.summaryDir, not args.overwrite)
 321     except exceptions.PiglitException:
 322         raise exceptions.PiglitFatalError(
 323             '{} already exists.\n'
 324             'use -o/--overwrite if you want to overwrite it.'.format(
 325                 args.summaryDir))
 326
 327     summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)