contrib/dg-extract-results.py

   1 #!/usr/bin/python
   2 #
   3 # Copyright (C) 2014-2024 Free Software Foundation, Inc.
   4 #
   5 # This script is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3, or (at your option)
   8 # any later version.
   9
  10 import sys
  11 import getopt
  12 import re
  13 import io
  14 from datetime import datetime
  15 from operator import attrgetter
  16
  17 # True if unrecognised lines should cause a fatal error.  Might want to turn
  18 # this on by default later.
  19 strict = False
  20
  21 # True if the order of .log segments should match the .sum file, false if
  22 # they should keep the original order.
  23 sort_logs = True
  24
  25 # A version of open() that is safe against whatever binary output
  26 # might be added to the log.
  27 def safe_open (filename):
  28     if sys.version_info >= (3, 0):
  29         return open (filename, 'r', errors = 'surrogateescape')
  30     return open (filename, 'r')
  31
  32 # Force stdout to handle escape sequences from a safe_open file.
  33 if sys.version_info >= (3, 0):
  34     sys.stdout = io.TextIOWrapper (sys.stdout.buffer,
  35                                    errors = 'surrogateescape')
  36
  37 class Named:
  38     def __init__ (self, name):
  39         self.name = name
  40
  41 class ToolRun (Named):
  42     def __init__ (self, name):
  43         Named.__init__ (self, name)
  44         # The variations run for this tool, mapped by --target_board name.
  45         self.variations = dict()
  46
  47     # Return the VariationRun for variation NAME.
  48     def get_variation (self, name):
  49         if name not in self.variations:
  50             self.variations[name] = VariationRun (name)
  51         return self.variations[name]
  52
  53 class VariationRun (Named):
  54     def __init__ (self, name):
  55         Named.__init__ (self, name)
  56         # A segment of text before the harness runs start, describing which
  57         # baseboard files were loaded for the target.
  58         self.header = None
  59         # The harnesses run for this variation, mapped by filename.
  60         self.harnesses = dict()
  61         # A list giving the number of times each type of result has
  62         # been seen.
  63         self.counts = []
  64
  65     # Return the HarnessRun for harness NAME.
  66     def get_harness (self, name):
  67         if name not in self.harnesses:
  68             self.harnesses[name] = HarnessRun (name)
  69         return self.harnesses[name]
  70
  71 class HarnessRun (Named):
  72     def __init__ (self, name):
  73         Named.__init__ (self, name)
  74         # Segments of text that make up the harness run, mapped by a test-based
  75         # key that can be used to order them.
  76         self.segments = dict()
  77         # Segments of text that make up the harness run but which have
  78         # no recognized test results.  These are typically harnesses that
  79         # are completely skipped for the target.
  80         self.empty = []
  81         # A list of results.  Each entry is a pair in which the first element
  82         # is a unique sorting key and in which the second is the full
  83         # PASS/FAIL line.
  84         self.results = []
  85
  86     # Add a segment of text to the harness run.  If the segment includes
  87     # test results, KEY is an example of one of them, and can be used to
  88     # combine the individual segments in order.  If the segment has no
  89     # test results (e.g. because the harness doesn't do anything for the
  90     # current configuration) then KEY is None instead.  In that case
  91     # just collect the segments in the order that we see them.
  92     def add_segment (self, key, segment):
  93         if key:
  94             assert key not in self.segments
  95             self.segments[key] = segment
  96         else:
  97             self.empty.append (segment)
  98
  99 class Segment:
 100     def __init__ (self, filename, start):
 101         self.filename = filename
 102         self.start = start
 103         self.lines = 0
 104
 105 class Prog:
 106     def __init__ (self):
 107         # The variations specified on the command line.
 108         self.variations = []
 109         # The variations seen in the input files.
 110         self.known_variations = set()
 111         # The tools specified on the command line.
 112         self.tools = []
 113         # Whether to create .sum rather than .log output.
 114         self.do_sum = True
 115         # Regexps used while parsing.
 116         self.test_run_re = re.compile (r'^Test run by (\S+) on (.*)$',
 117                                        re.IGNORECASE)
 118         self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$')
 119         self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED'
 120                                      r'|WARNING|ERROR|UNSUPPORTED|UNTESTED'
 121                                      r'|KFAIL|KPASS|PATH|DUPLICATE):\s*(.+)')
 122         self.completed_re = re.compile (r'.* completed at (.*)')
 123         # Pieces of text to write at the head of the output.
 124         # start_line is a pair in which the first element is a datetime
 125         # and in which the second is the associated 'Test Run By' line.
 126         self.start_line = None
 127         self.native_line = ''
 128         self.target_line = ''
 129         self.host_line = ''
 130         self.acats_premable = ''
 131         # Pieces of text to write at the end of the output.
 132         # end_line is like start_line but for the 'runtest completed' line.
 133         self.acats_failures = []
 134         self.version_output = ''
 135         self.end_line = None
 136         # Known summary types.
 137         self.count_names = [
 138             '# of DejaGnu errors\t\t',
 139             '# of expected passes\t\t',
 140             '# of unexpected failures\t',
 141             '# of unexpected successes\t',
 142             '# of expected failures\t\t',
 143             '# of unknown successes\t\t',
 144             '# of known failures\t\t',
 145             '# of untested testcases\t\t',
 146             '# of unresolved testcases\t',
 147             '# of unsupported tests\t\t',
 148             '# of paths in test names\t',
 149             '# of duplicate test names\t'
 150         ]
 151         self.runs = dict()
 152
 153     def usage (self):
 154         name = sys.argv[0]
 155         sys.stderr.write ('Usage: ' + name
 156                           + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ...
 157
 158     tool           The tool (e.g. g++, libffi) for which to create a
 159                    new test summary file.  If not specified then output
 160                    is created for all tools.
 161     variant-list   One or more test variant names.  If the list is
 162                    not specified then one is constructed from all
 163                    variants in the files for <tool>.
 164     sum-file       A test summary file with the format of those
 165                    created by runtest from DejaGnu.
 166     If -L is used, merge *.log files instead of *.sum.  In this
 167     mode the exact order of lines may not be preserved, just different
 168     Running *.exp chunks should be in correct order.
 169 ''')
 170         sys.exit (1)
 171
 172     def fatal (self, what, string):
 173         if not what:
 174             what = sys.argv[0]
 175         sys.stderr.write (what + ': ' + string + '\n')
 176         sys.exit (1)
 177
 178     # Parse the command-line arguments.
 179     def parse_cmdline (self):
 180         try:
 181             (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L')
 182             if len (self.files) == 0:
 183                 self.usage()
 184             for (option, value) in options:
 185                 if option == '-l':
 186                     self.variations.append (value)
 187                 elif option == '-t':
 188                     self.tools.append (value)
 189                 else:
 190                     self.do_sum = False
 191         except getopt.GetoptError as e:
 192             self.fatal (None, e.msg)
 193
 194     # Try to parse time string TIME, returning an arbitrary time on failure.
 195     # Getting this right is just a nice-to-have so failures should be silent.
 196     def parse_time (self, time):
 197         try:
 198             return datetime.strptime (time, '%c')
 199         except ValueError:
 200             return datetime.now()
 201
 202     # Parse an integer and abort on failure.
 203     def parse_int (self, filename, value):
 204         try:
 205             return int (value)
 206         except ValueError:
 207             self.fatal (filename, 'expected an integer, got: ' + value)
 208
 209     # Return a list that represents no test results.
 210     def zero_counts (self):
 211         return [0 for x in self.count_names]
 212
 213     # Return the ToolRun for tool NAME.
 214     def get_tool (self, name):
 215         if name not in self.runs:
 216             self.runs[name] = ToolRun (name)
 217         return self.runs[name]
 218
 219     # Add the result counts in list FROMC to TOC.
 220     def accumulate_counts (self, toc, fromc):
 221         for i in range (len (self.count_names)):
 222             toc[i] += fromc[i]
 223
 224     # Parse the list of variations after 'Schedule of variations:'.
 225     # Return the number seen.
 226     def parse_variations (self, filename, file):
 227         num_variations = 0
 228         while True:
 229             line = file.readline()
 230             if line == '':
 231                 self.fatal (filename, 'could not parse variation list')
 232             if line == '\n':
 233                 break
 234             self.known_variations.add (line.strip())
 235             num_variations += 1
 236         return num_variations
 237
 238     # Parse from the first line after 'Running target ...' to the end
 239     # of the run's summary.
 240     def parse_run (self, filename, file, tool, variation, num_variations):
 241         header = None
 242         harness = None
 243         segment = None
 244         final_using = 0
 245         has_warning = 0
 246
 247         # If this is the first run for this variation, add any text before
 248         # the first harness to the header.
 249         if not variation.header:
 250             segment = Segment (filename, file.tell())
 251             variation.header = segment
 252
 253         # Parse the rest of the summary (the '# of ' lines).
 254         if len (variation.counts) == 0:
 255             variation.counts = self.zero_counts()
 256
 257         # Parse up until the first line of the summary.
 258         if num_variations == 1:
 259             end = '\t\t=== ' + tool.name + ' Summary ===\n'
 260         else:
 261             end = ('\t\t=== ' + tool.name + ' Summary for '
 262                    + variation.name + ' ===\n')
 263         while True:
 264             line = file.readline()
 265             if line == '':
 266                 self.fatal (filename, 'no recognised summary line')
 267             if line == end:
 268                 break
 269
 270             # Look for the start of a new harness.
 271             if line.startswith ('Running ') and line.endswith (' ...\n'):
 272                 # Close off the current harness segment, if any.
 273                 if harness:
 274                     segment.lines -= final_using
 275                     harness.add_segment (first_key, segment)
 276                 name = line[len ('Running '):-len(' ...\n')]
 277                 harness = variation.get_harness (name)
 278                 segment = Segment (filename, file.tell())
 279                 first_key = None
 280                 final_using = 0
 281                 continue
 282
 283             # Record test results.  Associate the first test result with
 284             # the harness segment, so that if a run for a particular harness
 285             # has been split up, we can reassemble the individual segments
 286             # in a sensible order.
 287             #
 288             # dejagnu sometimes issues warnings about the testing environment
 289             # before running any tests.  Treat them as part of the header
 290             # rather than as a test result.
 291             match = self.result_re.match (line)
 292             if match and (harness or not line.startswith ('WARNING:')):
 293                 if not harness:
 294                     self.fatal (filename, 'saw test result before harness name')
 295                 name = match.group (2)
 296                 # Ugly hack to get the right order for gfortran.
 297                 if name.startswith ('gfortran.dg/g77/'):
 298                     name = 'h' + name
 299                 # If we have a time out warning, make sure it appears
 300                 # before the following testcase diagnostic: we insert
 301                 # the testname before 'program' so that sort faces a
 302                 # list of testnames.
 303                 if line.startswith ('WARNING: program timed out'):
 304                   has_warning = 1
 305                 else:
 306                   if has_warning == 1:
 307                       key = (name, len (harness.results))
 308                       myline = 'WARNING: %s program timed out.\n' % name
 309                       harness.results.append ((key, myline))
 310                       has_warning = 0
 311                   key = (name, len (harness.results))
 312                   harness.results.append ((key, line))
 313                   if not first_key and sort_logs:
 314                       first_key = key
 315                 if line.startswith ('ERROR: (DejaGnu)'):
 316                     for i in range (len (self.count_names)):
 317                         if 'DejaGnu errors' in self.count_names[i]:
 318                             variation.counts[i] += 1
 319                             break
 320
 321             # 'Using ...' lines are only interesting in a header.  Splitting
 322             # the test up into parallel runs leads to more 'Using ...' lines
 323             # than there would be in a single log.
 324             if line.startswith ('Using '):
 325                 final_using += 1
 326             else:
 327                 final_using = 0
 328
 329             # Add other text to the current segment, if any.
 330             if segment:
 331                 segment.lines += 1
 332
 333         # Close off the final harness segment, if any.
 334         if harness:
 335             segment.lines -= final_using
 336             harness.add_segment (first_key, segment)
 337
 338         while True:
 339             before = file.tell()
 340             line = file.readline()
 341             if line == '':
 342                 break
 343             if line == '\n':
 344                 continue
 345             if not line.startswith ('# '):
 346                 file.seek (before)
 347                 break
 348             found = False
 349             for i in range (len (self.count_names)):
 350                 if line.startswith (self.count_names[i]):
 351                     count = line[len (self.count_names[i]):-1].strip()
 352                     variation.counts[i] += self.parse_int (filename, count)
 353                     found = True
 354                     break
 355             if not found:
 356                 self.fatal (filename, 'unknown test result: ' + line[:-1])
 357
 358     # Parse an acats run, which uses a different format from dejagnu.
 359     # We have just skipped over '=== acats configuration ==='.
 360     def parse_acats_run (self, filename, file):
 361         # Parse the preamble, which describes the configuration and logs
 362         # the creation of support files.
 363         record = (self.acats_premable == '')
 364         if record:
 365             self.acats_premable = '\t\t=== acats configuration ===\n'
 366         while True:
 367             line = file.readline()
 368             if line == '':
 369                 self.fatal (filename, 'could not parse acats preamble')
 370             if line == '\t\t=== acats tests ===\n':
 371                 break
 372             if record:
 373                 self.acats_premable += line
 374
 375         # Parse the test results themselves, using a dummy variation name.
 376         tool = self.get_tool ('acats')
 377         variation = tool.get_variation ('none')
 378         self.parse_run (filename, file, tool, variation, 1)
 379
 380         # Parse the failure list.
 381         while True:
 382             before = file.tell()
 383             line = file.readline()
 384             if line.startswith ('*** FAILURES: '):
 385                 self.acats_failures.append (line[len ('*** FAILURES: '):-1])
 386                 continue
 387             file.seek (before)
 388             break
 389
 390     # Parse the final summary at the end of a log in order to capture
 391     # the version output that follows it.
 392     def parse_final_summary (self, filename, file):
 393         record = (self.version_output == '')
 394         while True:
 395             line = file.readline()
 396             if line == '':
 397                 break
 398             if line.startswith ('# of '):
 399                 continue
 400             if record:
 401                 self.version_output += line
 402             if line == '\n':
 403                 break
 404
 405     # Parse a .log or .sum file.
 406     def parse_file (self, filename, file):
 407         tool = None
 408         target = None
 409         num_variations = 1
 410         while True:
 411             line = file.readline()
 412             if line == '':
 413                 return
 414
 415             # Parse the list of variations, which comes before the test
 416             # runs themselves.
 417             if line.startswith ('Schedule of variations:'):
 418                 num_variations = self.parse_variations (filename, file)
 419                 continue
 420
 421             # Parse a testsuite run for one tool/variation combination.
 422             if line.startswith ('Running target '):
 423                 name = line[len ('Running target '):-1]
 424                 if not tool:
 425                     self.fatal (filename, 'could not parse tool name')
 426                 if name not in self.known_variations:
 427                     self.fatal (filename, 'unknown target: ' + name)
 428                 self.parse_run (filename, file, tool,
 429                                 tool.get_variation (name),
 430                                 num_variations)
 431                 # If there is only one variation then there is no separate
 432                 # summary for it.  Record any following version output.
 433                 if num_variations == 1:
 434                     self.parse_final_summary (filename, file)
 435                 continue
 436
 437             # Parse the start line.  In the case where several files are being
 438             # parsed, pick the one with the earliest time.
 439             match = self.test_run_re.match (line)
 440             if match:
 441                 time = self.parse_time (match.group (2))
 442                 if not self.start_line or self.start_line[0] > time:
 443                     self.start_line = (time, line)
 444                 continue
 445
 446             # Parse the form used for native testing.
 447             if line.startswith ('Native configuration is '):
 448                 self.native_line = line
 449                 continue
 450
 451             # Parse the target triplet.
 452             if line.startswith ('Target is '):
 453                 self.target_line = line
 454                 continue
 455
 456             # Parse the host triplet.
 457             if line.startswith ('Host   is '):
 458                 self.host_line = line
 459                 continue
 460
 461             # Parse the acats premable.
 462             if line == '\t\t=== acats configuration ===\n':
 463                 self.parse_acats_run (filename, file)
 464                 continue
 465
 466             # Parse the tool name.
 467             match = self.tool_re.match (line)
 468             if match:
 469                 tool = self.get_tool (match.group (1))
 470                 continue
 471
 472             # Skip over the final summary (which we instead create from
 473             # individual runs) and parse the version output.
 474             if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n':
 475                 if file.readline() != '\n':
 476                     self.fatal (filename, 'expected blank line after summary')
 477                 self.parse_final_summary (filename, file)
 478                 continue
 479
 480             # Parse the completion line.  In the case where several files
 481             # are being parsed, pick the one with the latest time.
 482             match = self.completed_re.match (line)
 483             if match:
 484                 time = self.parse_time (match.group (1))
 485                 if not self.end_line or self.end_line[0] < time:
 486                     self.end_line = (time, line)
 487                 continue
 488
 489             # Sanity check to make sure that important text doesn't get
 490             # dropped accidentally.
 491             if strict and line.strip() != '':
 492                 self.fatal (filename, 'unrecognised line: ' + line[:-1])
 493
 494     # Output a segment of text.
 495     def output_segment (self, segment):
 496         with safe_open (segment.filename) as file:
 497             file.seek (segment.start)
 498             for i in range (segment.lines):
 499                 sys.stdout.write (file.readline())
 500
 501     # Output a summary giving the number of times each type of result has
 502     # been seen.
 503     def output_summary (self, tool, counts):
 504         for i in range (len (self.count_names)):
 505             name = self.count_names[i]
 506             # dejagnu only prints result types that were seen at least once,
 507             # but acats always prints a number of unexpected failures.
 508             if (counts[i] > 0
 509                 or (tool.name == 'acats'
 510                     and name.startswith ('# of unexpected failures'))):
 511                 sys.stdout.write ('%s%d\n' % (name, counts[i]))
 512
 513     # Output unified .log or .sum information for a particular variation,
 514     # with a summary at the end.
 515     def output_variation (self, tool, variation):
 516         self.output_segment (variation.header)
 517         for harness in sorted (variation.harnesses.values(),
 518                                key = attrgetter ('name')):
 519             sys.stdout.write ('Running ' + harness.name + ' ...\n')
 520             if self.do_sum:
 521                 harness.results.sort()
 522                 for (key, line) in harness.results:
 523                     sys.stdout.write (line)
 524             else:
 525                 # Rearrange the log segments into test order (but without
 526                 # rearranging text within those segments).
 527                 for key in sorted (harness.segments.keys()):
 528                     self.output_segment (harness.segments[key])
 529                 for segment in harness.empty:
 530                     self.output_segment (segment)
 531         if len (self.variations) > 1:
 532             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for '
 533                               + variation.name + ' ===\n\n')
 534             self.output_summary (tool, variation.counts)
 535
 536     # Output unified .log or .sum information for a particular tool,
 537     # with a summary at the end.
 538     def output_tool (self, tool):
 539         counts = self.zero_counts()
 540         if tool.name == 'acats':
 541             # acats doesn't use variations, so just output everything.
 542             # It also has a different approach to whitespace.
 543             sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n')
 544             for variation in tool.variations.values():
 545                 self.output_variation (tool, variation)
 546                 self.accumulate_counts (counts, variation.counts)
 547             sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n')
 548         else:
 549             # Output the results in the usual dejagnu runtest format.
 550             sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n'
 551                               'Schedule of variations:\n')
 552             for name in self.variations:
 553                 if name in tool.variations:
 554                     sys.stdout.write ('    ' + name + '\n')
 555             sys.stdout.write ('\n')
 556             for name in self.variations:
 557                 if name in tool.variations:
 558                     variation = tool.variations[name]
 559                     sys.stdout.write ('Running target '
 560                                       + variation.name + '\n')
 561                     self.output_variation (tool, variation)
 562                     self.accumulate_counts (counts, variation.counts)
 563             sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n')
 564         self.output_summary (tool, counts)
 565
 566     def main (self):
 567         self.parse_cmdline()
 568         try:
 569             # Parse the input files.
 570             for filename in self.files:
 571                 with safe_open (filename) as file:
 572                     self.parse_file (filename, file)
 573
 574             # Decide what to output.
 575             if len (self.variations) == 0:
 576                 self.variations = sorted (self.known_variations)
 577             else:
 578                 for name in self.variations:
 579                     if name not in self.known_variations:
 580                         self.fatal (None, 'no results for ' + name)
 581             if len (self.tools) == 0:
 582                 self.tools = sorted (self.runs.keys())
 583
 584             # Output the header.
 585             if self.start_line:
 586                 sys.stdout.write (self.start_line[1])
 587             sys.stdout.write (self.native_line)
 588             sys.stdout.write (self.target_line)
 589             sys.stdout.write (self.host_line)
 590             sys.stdout.write (self.acats_premable)
 591
 592             # Output the main body.
 593             for name in self.tools:
 594                 if name not in self.runs:
 595                     self.fatal (None, 'no results for ' + name)
 596                 self.output_tool (self.runs[name])
 597
 598             # Output the footer.
 599             if len (self.acats_failures) > 0:
 600                 sys.stdout.write ('*** FAILURES: '
 601                                   + ' '.join (self.acats_failures) + '\n')
 602             sys.stdout.write (self.version_output)
 603             if self.end_line:
 604                 sys.stdout.write (self.end_line[1])
 605         except IOError as e:
 606             self.fatal (e.filename, e.strerror)
 607
 608 Prog().main()