contrib/dg-extract-results.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (C) 2014 Free Software Foundation, Inc.
   4 #
   5 # This script is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3, or (at your option)
   8 # any later version.
   9
  10 import sys
  11 import getopt
  12 import re
  13 import io
  14 from datetime import datetime
  15 from operator import attrgetter
  16
  17 # True if unrecognised lines should cause a fatal error.  Might want to turn
  18 # this on by default later.
  19 strict = False
  20
  21 # True if the order of .log segments should match the .sum file, false if
  22 # they should keep the original order.
  23 sort_logs = True
  24
  25 # A version of open() that is safe against whatever binary output
  26 # might be added to the log.
  27 def safe_open (filename):
  28     if sys.version_info >= (3, 0):
  29         return open (filename, 'r', errors = 'surrogateescape')
  30     return open (filename, 'r')
  31
  32 # Force stdout to handle escape sequences from a safe_open file.
  33 if sys.version_info >= (3, 0):
  34     sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
  35                                    errors = 'surrogateescape')
  36
  37 class Named:
  38     def __init__ (self, name):
  39         self.name = name
  40
  41 class ToolRun (Named):
  42     def __init__ (self, name):
  43         Named.__init__ (self, name)
  44         # The variations run for this tool, mapped by --target_board name.
  45         self.variations = dict()
  46
  47     # Return the VariationRun for variation NAME.
  48     def get_variation (self, name):
  49         if name not in self.variations:
  50             self.variations[name] = VariationRun (name)
  51         return self.variations[name]
  52
  53 class VariationRun (Named):
  54     def __init__ (self, name):
  55         Named.__init__ (self, name)
  56         # A segment of text before the harness runs start, describing which
  57         # baseboard files were loaded for the target.
  58         self.header = None
  59         # The harnesses run for this variation, mapped by filename.
  60         self.harnesses = dict()
  61         # A list giving the number of times each type of result has
  62         # been seen.
  63         self.counts = []
  64
  65     # Return the HarnessRun for harness NAME.
  66     def get_harness (self, name):
  67         if name not in self.harnesses:
  68             self.harnesses[name] = HarnessRun (name)
  69         return self.harnesses[name]
  70
  71 class HarnessRun (Named):
  72     def __init__ (self, name):
  73         Named.__init__ (self, name)
  74         # Segments of text that make up the harness run, mapped by a test-based
  75         # key that can be used to order them.
  76         self.segments = dict()
  77         # Segments of text that make up the harness run but which have
  78         # no recognized test results.  These are typically harnesses that
  79         # are completely skipped for the target.
  80         self.empty = []
  81         # A list of results.  Each entry is a pair in which the first element
  82         # is a unique sorting key and in which the second is the full
  83         # PASS/FAIL line.
  84         self.results = []
  85
  86     # Add a segment of text to the harness run.  If the segment includes
  87     # test results, KEY is an example of one of them, and can be used to
  88     # combine the individual segments in order.  If the segment has no
  89     # test results (e.g. because the harness doesn't do anything for the
  90     # current configuration) then KEY is None instead.  In that case
  91     # just collect the segments in the order that we see them.
  92     def add_segment (self, key, segment):
  93         if key:
  94             assert key not in self.segments
  95             self.segments[key] = segment
  96         else:
  97             self.empty.append (segment)
  98
  99 class Segment:
 100     def __init__ (self, filename, start):
 101         self.filename = filename
 102         self.start = start
 103         self.lines = 0
 104
 105 class Prog:
 106     def __init__ (self):
 107         # The variations specified on the command line.
 108         self.variations = []
 109         # The variations seen in the input files.
 110         self.known_variations = set()
 111         # The tools specified on the command line.
 112         self.tools = []
 113         # Whether to create .sum rather than .log output.
 114         self.do_sum = True
 115         # Regexps used while parsing.
 116         self.test_run_re = re.compile (r'^Test Run By (\S+) on (.*)$')
 117         self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
 118         self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
 119                                      r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
 120                                      r'|KFAIL):\s*(.+)')
 121         self.completed_re = re.compile (r'.* completed at (.*)')
 122         # Pieces of text to write at the head of the output.
 123         # start_line is a pair in which the first element is a datetime
 124         # and in which the second is the associated 'Test Run By' line.
 125         self.start_line = None
 126         self.native_line = ''
 127         self.target_line = ''
 128         self.host_line = ''
 129         self.acats_premable = ''
 130         # Pieces of text to write at the end of the output.
 131         # end_line is like start_line but for the 'runtest completed' line.
 132         self.acats_failures = []
 133         self.version_output = ''
 134         self.end_line = None
 135         # Known summary types.
 136         self.count_names = [
 137             '# of DejaGnu errors\t\t',
 138             '# of expected passes\t\t',
 139             '# of unexpected failures\t',
 140             '# of unexpected successes\t',
 141             '# of expected failures\t\t',
 142             '# of unknown successes\t\t',
 143             '# of known failures\t\t',
 144             '# of untested testcases\t\t',
 145             '# of unresolved testcases\t',
 146             '# of unsupported tests\t\t'
 147         ]
 148         self.runs = dict()
 149
 150     def usage (self):
 151         name = sys.argv[0]
 152         sys.stderr.write ('Usage: ' + name
 153                           + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
 154
 155     tool           The tool (e.g. g++, libffi) for which to create a
 156                    new test summary file.  If not specified then output
 157                    is created for all tools.
 158     variant-list   One or more test variant names.  If the list is
 159                    not specified then one is constructed from all
 160                    variants in the files for <tool>.
 161     sum-file       A test summary file with the format of those
 162                    created by runtest from DejaGnu.
 163     If -L is used, merge *.log files instead of *.sum.  In this
 164     mode the exact order of lines may not be preserved, just different
 165     Running *.exp chunks should be in correct order.
 166 ''')
 167         sys.exit (1)
 168
 169     def fatal (self, what, string):
 170         if not what:
 171             what = sys.argv[0]
 172         sys.stderr.write (what + ': ' + string + '\n')
 173         sys.exit (1)
 174
 175     # Parse the command-line arguments.
 176     def parse_cmdline (self):
 177         try:
 178             (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
 179             if len (self.files) == 0:
 180                 self.usage()
 181             for (option, value) in options:
 182                 if option == '-l':
 183                     self.variations.append (value)
 184                 elif option == '-t':
 185                     self.tools.append (value)
 186                 else:
 187                     self.do_sum = False
 188         except getopt.GetoptError as e:
 189             self.fatal (None, e.msg)
 190
 191     # Try to parse time string TIME, returning an arbitrary time on failure.
 192     # Getting this right is just a nice-to-have so failures should be silent.
 193     def parse_time (self, time):
 194         try:
 195             return datetime.strptime (time, '%c')
 196         except ValueError:
 197             return datetime.now()
 198
 199     # Parse an integer and abort on failure.
 200     def parse_int (self, filename, value):
 201         try:
 202             return int (value)
 203         except ValueError:
 204             self.fatal (filename, 'expected an integer, got: ' + value)
 205
 206     # Return a list that represents no test results.
 207     def zero_counts (self):
 208         return [0 for x in self.count_names]
 209
 210     # Return the ToolRun for tool NAME.
 211     def get_tool (self, name):
 212         if name not in self.runs:
 213             self.runs[name] = ToolRun (name)
 214         return self.runs[name]
 215
 216     # Add the result counts in list FROMC to TOC.
 217     def accumulate_counts (self, toc, fromc):
 218         for i in range (len (self.count_names)):
 219             toc[i] += fromc[i]
 220
 221     # Parse the list of variations after 'Schedule of variations:'.
 222     # Return the number seen.
 223     def parse_variations (self, filename, file):
 224         num_variations = 0
 225         while True:
 226             line = file.readline()
 227             if line == '':
 228                 self.fatal (filename, 'could not parse variation list')
 229             if line == '\n':
 230                 break
 231             self.known_variations.add (line.strip())
 232             num_variations += 1
 233         return num_variations
 234
 235     # Parse from the first line after 'Running target ...' to the end
 236     # of the run's summary.
 237     def parse_run (self, filename, file, tool, variation, num_variations):
 238         header = None
 239         harness = None
 240         segment = None
 241         final_using = 0
 242
 243         # If this is the first run for this variation, add any text before
 244         # the first harness to the header.
 245         if not variation.header:
 246             segment = Segment (filename, file.tell())
 247             variation.header = segment
 248
 249         # Parse the rest of the summary (the '# of ' lines).
 250         if len (variation.counts) == 0:
 251             variation.counts = self.zero_counts()
 252
 253         # Parse up until the first line of the summary.
 254         if num_variations == 1:
 255             end = '\t\t=== ' + tool.name + ' Summary ===\n'
 256         else:
 257             end = ('\t\t=== ' + tool.name + ' Summary for '
 258                    + variation.name + ' ===\n')
 259         while True:
 260             line = file.readline()
 261             if line == '':
 262                 self.fatal (filename, 'no recognised summary line')
 263             if line == end:
 264                 break
 265
 266             # Look for the start of a new harness.
 267             if line.startswith ('Running ') and line.endswith (' ...\n'):
 268                 # Close off the current harness segment, if any.
 269                 if harness:
 270                     segment.lines -= final_using
 271                     harness.add_segment (first_key, segment)
 272                 name = line[len ('Running '):-len(' ...\n')]
 273                 harness = variation.get_harness (name)
 274                 segment = Segment (filename, file.tell())
 275                 first_key = None
 276                 final_using = 0
 277                 continue
 278
 279             # Record test results.  Associate the first test result with
 280             # the harness segment, so that if a run for a particular harness
 281             # has been split up, we can reassemble the individual segments
 282             # in a sensible order.
 283             #
 284             # dejagnu sometimes issues warnings about the testing environment
 285             # before running any tests.  Treat them as part of the header
 286             # rather than as a test result.
 287             match = self.result_re.match (line)
 288             if match and (harness or not line.startswith ('WARNING:')):
 289                 if not harness:
 290                     self.fatal (filename, 'saw test result before harness name')
 291                 name = match.group (2)
 292                 # Ugly hack to get the right order for gfortran.
 293                 if name.startswith ('gfortran.dg/g77/'):
 294                     name = 'h' + name
 295                 key = (name, len (harness.results))
 296                 harness.results.append ((key, line))
 297                 if not first_key and sort_logs:
 298                     first_key = key
 299                 if line.startswith ('ERROR: (DejaGnu)'):
 300                     for i in range (len (self.count_names)):
 301                         if 'DejaGnu errors' in self.count_names[i]:
 302                             variation.counts[i] += 1
 303                             break
 304
 305             # 'Using ...' lines are only interesting in a header.  Splitting
 306             # the test up into parallel runs leads to more 'Using ...' lines
 307             # than there would be in a single log.
 308             if line.startswith ('Using '):
 309                 final_using += 1
 310             else:
 311                 final_using = 0
 312
 313             # Add other text to the current segment, if any.
 314             if segment:
 315                 segment.lines += 1
 316
 317         # Close off the final harness segment, if any.
 318         if harness:
 319             segment.lines -= final_using
 320             harness.add_segment (first_key, segment)
 321
 322         while True:
 323             before = file.tell()
 324             line = file.readline()
 325             if line == '':
 326                 break
 327             if line == '\n':
 328                 continue
 329             if not line.startswith ('# '):
 330                 file.seek (before)
 331                 break
 332             found = False
 333             for i in range (len (self.count_names)):
 334                 if line.startswith (self.count_names[i]):
 335                     count = line[len (self.count_names[i]):-1].strip()
 336                     variation.counts[i] += self.parse_int (filename, count)
 337                     found = True
 338                     break
 339             if not found:
 340                 self.fatal (filename, 'unknown test result: ' + line[:-1])
 341
 342     # Parse an acats run, which uses a different format from dejagnu.
 343     # We have just skipped over '=== acats configuration ==='.
 344     def parse_acats_run (self, filename, file):
 345         # Parse the preamble, which describes the configuration and logs
 346         # the creation of support files.
 347         record = (self.acats_premable == '')
 348         if record:
 349             self.acats_premable = '\t\t=== acats configuration ===\n'
 350         while True:
 351             line = file.readline()
 352             if line == '':
 353                 self.fatal (filename, 'could not parse acats preamble')
 354             if line == '\t\t=== acats tests ===\n':
 355                 break
 356             if record:
 357                 self.acats_premable += line
 358
 359         # Parse the test results themselves, using a dummy variation name.
 360         tool = self.get_tool ('acats')
 361         variation = tool.get_variation ('none')
 362         self.parse_run (filename, file, tool, variation, 1)
 363
 364         # Parse the failure list.
 365         while True:
 366             before = file.tell()
 367             line = file.readline()
 368             if line.startswith ('*** FAILURES: '):
 369                 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
 370                 continue
 371             file.seek (before)
 372             break
 373
 374     # Parse the final summary at the end of a log in order to capture
 375     # the version output that follows it.
 376     def parse_final_summary (self, filename, file):
 377         record = (self.version_output == '')
 378         while True:
 379             line = file.readline()
 380             if line == '':
 381                 break
 382             if line.startswith ('# of '):
 383                 continue
 384             if record:
 385                 self.version_output += line
 386             if line == '\n':
 387                 break
 388
 389     # Parse a .log or .sum file.
 390     def parse_file (self, filename, file):
 391         tool = None
 392         target = None
 393         num_variations = 1
 394         while True:
 395             line = file.readline()
 396             if line == '':
 397                 return
 398
 399             # Parse the list of variations, which comes before the test
 400             # runs themselves.
 401             if line.startswith ('Schedule of variations:'):
 402                 num_variations = self.parse_variations (filename, file)
 403                 continue
 404
 405             # Parse a testsuite run for one tool/variation combination.
 406             if line.startswith ('Running target '):
 407                 name = line[len ('Running target '):-1]
 408                 if not tool:
 409                     self.fatal (filename, 'could not parse tool name')
 410                 if name not in self.known_variations:
 411                     self.fatal (filename, 'unknown target: ' + name)
 412                 self.parse_run (filename, file, tool,
 413                                 tool.get_variation (name),
 414                                 num_variations)
 415                 # If there is only one variation then there is no separate
 416                 # summary for it.  Record any following version output.
 417                 if num_variations == 1:
 418                     self.parse_final_summary (filename, file)
 419                 continue
 420
 421             # Parse the start line.  In the case where several files are being
 422             # parsed, pick the one with the earliest time.
 423             match = self.test_run_re.match (line)
 424             if match:
 425                 time = self.parse_time (match.group (2))
 426                 if not self.start_line or self.start_line[0] > time:
 427                     self.start_line = (time, line)
 428                 continue
 429
 430             # Parse the form used for native testing.
 431             if line.startswith ('Native configuration is '):
 432                 self.native_line = line
 433                 continue
 434
 435             # Parse the target triplet.
 436             if line.startswith ('Target is '):
 437                 self.target_line = line
 438                 continue
 439
 440             # Parse the host triplet.
 441             if line.startswith ('Host   is '):
 442                 self.host_line = line
 443                 continue
 444
 445             # Parse the acats premable.
 446             if line == '\t\t=== acats configuration ===\n':
 447                 self.parse_acats_run (filename, file)
 448                 continue
 449
 450             # Parse the tool name.
 451             match = self.tool_re.match (line)
 452             if match:
 453                 tool = self.get_tool (match.group (1))
 454                 continue
 455
 456             # Skip over the final summary (which we instead create from
 457             # individual runs) and parse the version output.
 458             if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
 459                 if file.readline() != '\n':
 460                     self.fatal (filename, 'expected blank line after summary')
 461                 self.parse_final_summary (filename, file)
 462                 continue
 463
 464             # Parse the completion line.  In the case where several files
 465             # are being parsed, pick the one with the latest time.
 466             match = self.completed_re.match (line)
 467             if match:
 468                 time = self.parse_time (match.group (1))
 469                 if not self.end_line or self.end_line[0] < time:
 470                     self.end_line = (time, line)
 471                 continue
 472
 473             # Sanity check to make sure that important text doesn't get
 474             # dropped accidentally.
 475             if strict and line.strip() != '':
 476                 self.fatal (filename, 'unrecognised line: ' + line[:-1])
 477
 478     # Output a segment of text.
 479     def output_segment (self, segment):
 480         with safe_open (segment.filename) as file:
 481             file.seek (segment.start)
 482             for i in range (segment.lines):
 483                 sys.stdout.write (file.readline())
 484
 485     # Output a summary giving the number of times each type of result has
 486     # been seen.
 487     def output_summary (self, tool, counts):
 488         for i in range (len (self.count_names)):
 489             name = self.count_names[i]
 490             # dejagnu only prints result types that were seen at least once,
 491             # but acats always prints a number of unexpected failures.
 492             if (counts[i] > 0
 493                 or (tool.name == 'acats'
 494                     and name.startswith ('# of unexpected failures'))):
 495                 sys.stdout.write ('%s%d\n' % (name, counts[i]))
 496
 497     # Output unified .log or .sum information for a particular variation,
 498     # with a summary at the end.
 499     def output_variation (self, tool, variation):
 500         self.output_segment (variation.header)
 501         for harness in sorted (variation.harnesses.values(),
 502                                key = attrgetter ('name')):
 503             sys.stdout.write ('Running ' + harness.name + ' ...\n')
 504             if self.do_sum:
 505                 harness.results.sort()
 506                 for (key, line) in harness.results:
 507                     sys.stdout.write (line)
 508             else:
 509                 # Rearrange the log segments into test order (but without
 510                 # rearranging text within those segments).
 511                 for key in sorted (harness.segments.keys()):
 512                     self.output_segment (harness.segments[key])
 513                 for segment in harness.empty:
 514                     self.output_segment (segment)
 515         if len (self.variations) > 1:
 516             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
 517                               + variation.name + ' ===\n\n')
 518             self.output_summary (tool, variation.counts)
 519
 520     # Output unified .log or .sum information for a particular tool,
 521     # with a summary at the end.
 522     def output_tool (self, tool):
 523         counts = self.zero_counts()
 524         if tool.name == 'acats':
 525             # acats doesn't use variations, so just output everything.
 526             # It also has a different approach to whitespace.
 527             sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
 528             for variation in tool.variations.values():
 529                 self.output_variation (tool, variation)
 530                 self.accumulate_counts (counts, variation.counts)
 531             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
 532         else:
 533             # Output the results in the usual dejagnu runtest format.
 534             sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
 535                               'Schedule of variations:\n')
 536             for name in self.variations:
 537                 if name in tool.variations:
 538                     sys.stdout.write ('    ' + name + '\n')
 539             sys.stdout.write ('\n')
 540             for name in self.variations:
 541                 if name in tool.variations:
 542                     variation = tool.variations[name]
 543                     sys.stdout.write ('Running target '
 544                                       + variation.name + '\n')
 545                     self.output_variation (tool, variation)
 546                     self.accumulate_counts (counts, variation.counts)
 547             sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
 548         self.output_summary (tool, counts)
 549
 550     def main (self):
 551         self.parse_cmdline()
 552         try:
 553             # Parse the input files.
 554             for filename in self.files:
 555                 with safe_open (filename) as file:
 556                     self.parse_file (filename, file)
 557
 558             # Decide what to output.
 559             if len (self.variations) == 0:
 560                 self.variations = sorted (self.known_variations)
 561             else:
 562                 for name in self.variations:
 563                     if name not in self.known_variations:
 564                         self.fatal (None, 'no results for ' + name)
 565             if len (self.tools) == 0:
 566                 self.tools = sorted (self.runs.keys())
 567
 568             # Output the header.
 569             if self.start_line:
 570                 sys.stdout.write (self.start_line[1])
 571             sys.stdout.write (self.native_line)
 572             sys.stdout.write (self.target_line)
 573             sys.stdout.write (self.host_line)
 574             sys.stdout.write (self.acats_premable)
 575
 576             # Output the main body.
 577             for name in self.tools:
 578                 if name not in self.runs:
 579                     self.fatal (None, 'no results for ' + name)
 580                 self.output_tool (self.runs[name])
 581
 582             # Output the footer.
 583             if len (self.acats_failures) > 0:
 584                 sys.stdout.write ('*** FAILURES: '
 585                                   + ' '.join (self.acats_failures) + '\n')
 586             sys.stdout.write (self.version_output)
 587             if self.end_line:
 588                 sys.stdout.write (self.end_line[1])
 589         except IOError as e:
 590             self.fatal (e.filename, e.strerror)
 591
 592 Prog().main()