polly/test/update_check.py

   1 #!/usr/bin/env python3
   2 # -*- coding: UTF-8 -*-
   3
   4 # Polly/LLVM update_check.py
   5 # Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.
   6
   7 import argparse
   8 import os
   9 import subprocess
  10 import shlex
  11 import re
  12
  13
  14 polly_src_dir = """@POLLY_SOURCE_DIR@"""
  15 polly_lib_dir = """@POLLY_LIB_DIR@"""
  16 shlibext = """@LLVM_SHLIBEXT@"""
  17 llvm_tools_dir = """@LLVM_TOOLS_DIR@"""
  18 llvm_polly_link_into_tools = not """@LLVM_POLLY_LINK_INTO_TOOLS@""".lower() in {
  19     "",
  20     "0",
  21     "n",
  22     "no",
  23     "off",
  24     "false",
  25     "notfound",
  26     "llvm_polly_link_into_tools-notfound",
  27 }
  28
  29 runre = re.compile(r"\s*\;\s*RUN\s*\:(?P<tool>.*)")
  30 filecheckre = re.compile(r"\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)")
  31 emptyline = re.compile(r"\s*(\;\s*)?")
  32 commentline = re.compile(r"\s*(\;.*)?")
  33
  34
  35 def ltrim_emptylines(lines, meta=None):
  36     while len(lines) and emptyline.fullmatch(lines[0]):
  37         del lines[0]
  38         if meta is not None:
  39             del meta[0]
  40
  41
  42 def rtrim_emptylines(lines):
  43     while len(lines) and emptyline.fullmatch(lines[-1]):
  44         del lines[-1]
  45
  46
  47 def trim_emptylines(lines):
  48     ltrim_emptylines(lines)
  49     rtrim_emptylines(lines)
  50
  51
  52 def complete_exename(path, filename):
  53     complpath = os.path.join(path, filename)
  54     if os.path.isfile(complpath):
  55         return complpath
  56     elif os.path.isfile(complpath + ".exe"):
  57         return complpath + ".exe"
  58     return filename
  59
  60
  61 def indention(line):
  62     for i, c in enumerate(line):
  63         if c != " " and c != "\t":
  64             return i
  65     return None
  66
  67
  68 def common_indent(lines):
  69     indentions = (indention(line) for line in lines)
  70     indentions = (indent for indent in indentions if indent is not None)
  71     return min(indentions, default=0)
  72
  73
  74 funcre = re.compile(r"^    Function: \S*$")
  75 regionre = re.compile(r"^    Region: \S*$")
  76 depthre = re.compile(r"^    Max Loop Depth: .*")
  77 paramre = re.compile(r"    [0-9a-z-A-Z_]+\: .*")
  78
  79
  80 def classyfier1(lines):
  81     i = iter(lines)
  82     line = i.__next__()
  83     while True:
  84         if line.startswith(
  85             "Printing analysis 'Polly - Calculate dependences' for region: "
  86         ):
  87             yield {"PrintingDependenceInfo"}
  88         elif line.startswith("remark: "):
  89             yield {"Remark"}
  90         elif funcre.fullmatch(line):
  91             yield {"Function"}
  92         elif regionre.fullmatch(line):
  93             yield {"Region"}
  94         elif depthre.fullmatch(line):
  95             yield {"MaxLoopDepth"}
  96         elif line == "    Invariant Accesses: {":
  97             while True:
  98                 yield {"InvariantAccesses"}
  99                 if line == "    }":
 100                     break
 101                 line = i.__next__()
 102         elif line == "    Context:":
 103             yield {"Context"}
 104             line = i.__next__()
 105             yield {"Context"}
 106         elif line == "    Assumed Context:":
 107             yield {"AssumedContext"}
 108             line = i.__next__()
 109             yield {"AssumedContext"}
 110         elif line == "    Invalid Context:":
 111             yield {"InvalidContext"}
 112             line = i.__next__()
 113             yield {"InvalidContext"}
 114         elif line == "    Boundary Context:":
 115             yield {"BoundaryContext"}
 116             line = i.__next__()
 117             yield {"BoundaryContext"}
 118             line = i.__next__()
 119             while paramre.fullmatch(line):
 120                 yield {"Param"}
 121                 line = i.__next__()
 122             continue
 123         elif line == "    Arrays {":
 124             while True:
 125                 yield {"Arrays"}
 126                 if line == "    }":
 127                     break
 128                 line = i.__next__()
 129         elif line == "    Arrays (Bounds as pw_affs) {":
 130             while True:
 131                 yield {"PwAffArrays"}
 132                 if line == "    }":
 133                     break
 134                 line = i.__next__()
 135         elif line.startswith("    Alias Groups ("):
 136             while True:
 137                 yield {"AliasGroups"}
 138                 line = i.__next__()
 139                 if not line.startswith("        "):
 140                     break
 141             continue
 142         elif line == "    Statements {":
 143             while True:
 144                 yield {"Statements"}
 145                 if line == "    }":
 146                     break
 147                 line = i.__next__()
 148         elif line == "    RAW dependences:":
 149             yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}
 150             line = i.__next__()
 151             while line.startswith("        "):
 152                 yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}
 153                 line = i.__next__()
 154             continue
 155         elif line == "    WAR dependences:":
 156             yield {"WARDep", "BasicDep", "Dep", "DepInfo"}
 157             line = i.__next__()
 158             while line.startswith("        "):
 159                 yield {"WARDep", "BasicDep", "Dep", "DepInfo"}
 160                 line = i.__next__()
 161             continue
 162         elif line == "    WAW dependences:":
 163             yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}
 164             line = i.__next__()
 165             while line.startswith("        "):
 166                 yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}
 167                 line = i.__next__()
 168             continue
 169         elif line == "    Reduction dependences:":
 170             yield {"RedDep", "Dep", "DepInfo"}
 171             line = i.__next__()
 172             while line.startswith("        "):
 173                 yield {"RedDep", "Dep", "DepInfo"}
 174                 line = i.__next__()
 175             continue
 176         elif line == "    Transitive closure of reduction dependences:":
 177             yield {"TransitiveClosureDep", "DepInfo"}
 178             line = i.__next__()
 179             while line.startswith("        "):
 180                 yield {"TransitiveClosureDep", "DepInfo"}
 181                 line = i.__next__()
 182             continue
 183         elif line.startswith("New access function '"):
 184             yield {"NewAccessFunction"}
 185         elif line == "Schedule before flattening {":
 186             while True:
 187                 yield {"ScheduleBeforeFlattening"}
 188                 if line == "}":
 189                     break
 190                 line = i.__next__()
 191         elif line == "Schedule after flattening {":
 192             while True:
 193                 yield {"ScheduleAfterFlattening"}
 194                 if line == "}":
 195                     break
 196                 line = i.__next__()
 197         else:
 198             yield set()
 199         line = i.__next__()
 200
 201
 202 def classyfier2(lines):
 203     i = iter(lines)
 204     line = i.__next__()
 205     while True:
 206         if funcre.fullmatch(line):
 207             while line.startswith("    "):
 208                 yield {"FunctionDetail"}
 209                 line = i.__next__()
 210             continue
 211         elif line.startswith(
 212             "Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "
 213         ):
 214             yield {"PrintingIslAst"}
 215             line = i.__next__()
 216             while not line.startswith("Printing analysis"):
 217                 yield {"AstDetail"}
 218                 line = i.__next__()
 219             continue
 220         else:
 221             yield set()
 222         line = i.__next__()
 223
 224
 225 replrepl = {
 226     "{{": "{{[{][{]}}",
 227     "}}": "{{[}][}]}}",
 228     "[[": r"{{\[\[}}",
 229     "]]": r"{{\]\]}}",
 230 }
 231 replre = re.compile("|".join(re.escape(k) for k in replrepl.keys()))
 232
 233
 234 def main():
 235     parser = argparse.ArgumentParser(description="Update CHECK lines")
 236     parser.add_argument(
 237         "testfile", help="File to update (absolute or relative to --testdir)"
 238     )
 239     parser.add_argument(
 240         "--check-style",
 241         choices=["CHECK", "CHECK-NEXT"],
 242         default="CHECK-NEXT",
 243         help="What kind of checks lines to generate",
 244     )
 245     parser.add_argument(
 246         "--check-position",
 247         choices=["end", "before-content", "autodetect"],
 248         default="autodetect",
 249         help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there",
 250     )
 251     parser.add_argument(
 252         "--check-include",
 253         action="append",
 254         default=[],
 255         help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)",
 256     )
 257     parser.add_argument(
 258         "--check-label-include",
 259         action="append",
 260         default=[],
 261         help="Use CHECK-LABEL for these includes",
 262     )
 263     parser.add_argument(
 264         "--check-part-newline",
 265         action="store_true",
 266         help="Add empty line between different check parts",
 267     )
 268     parser.add_argument(
 269         "--prefix-only",
 270         action="append",
 271         default=None,
 272         help="Update only these prefixes (default: all)",
 273     )
 274     parser.add_argument("--bindir", help="Location of the opt program")
 275     parser.add_argument("--testdir", help="Root dir for unit tests")
 276     parser.add_argument(
 277         "--inplace", "-i", action="store_true", help="Replace input file"
 278     )
 279     parser.add_argument("--output", "-o", help="Write changed input to this file")
 280     known = parser.parse_args()
 281
 282     if not known.inplace and known.output is None:
 283         print("Must specify what to do with output (--output or --inplace)")
 284         exit(1)
 285     if known.inplace and known.output is not None:
 286         print("--inplace and --output are mutually exclusive")
 287         exit(1)
 288
 289     outfile = known.output
 290
 291     filecheckparser = argparse.ArgumentParser(add_help=False)
 292     filecheckparser.add_argument("-check-prefix", "--check-prefix", default="CHECK")
 293
 294     filename = known.testfile
 295     for dir in [".", known.testdir, os.path.join(polly_src_dir, "test"), polly_src_dir]:
 296         if not dir:
 297             continue
 298         testfilename = os.path.join(dir, filename)
 299         if os.path.isfile(testfilename):
 300             filename = testfilename
 301             break
 302
 303     if known.inplace:
 304         outfile = filename
 305
 306     allchecklines = []
 307     checkprefixes = []
 308
 309     with open(filename, "r") as file:
 310         oldlines = [line.rstrip("\r\n") for line in file.readlines()]
 311
 312     runlines = []
 313     for line in oldlines:
 314         m = runre.match(line)
 315         if m:
 316             runlines.append(m.group("tool"))
 317
 318     continuation = ""
 319     newrunlines = []
 320     for line in runlines:
 321         if line.endswith("\\"):
 322             continuation += line[:-2] + " "
 323         else:
 324             newrunlines.append(continuation + line)
 325             continuation = ""
 326     if continuation:
 327         newrunlines.append(continuation)
 328
 329     for line in newrunlines:
 330         m = filecheckre.match(line)
 331         if not m:
 332             continue
 333
 334         tool, filecheck = m.group("tool", "filecheck")
 335         filecheck = shlex.split(filecheck)
 336         tool = shlex.split(tool)
 337         if known.bindir is not None:
 338             tool[0] = complete_exename(known.bindir, tool[0])
 339         if os.path.isdir(llvm_tools_dir):
 340             tool[0] = complete_exename(llvm_tools_dir, tool[0])
 341         check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix
 342         if known.prefix_only is not None and not check_prefix in known.prefix_only:
 343             continue
 344         if check_prefix in checkprefixes:
 345             continue
 346         checkprefixes.append(check_prefix)
 347
 348         newtool = []
 349         optstderr = None
 350         for toolarg in tool:
 351             toolarg = toolarg.replace("%s", filename)
 352             toolarg = toolarg.replace("%S", os.path.dirname(filename))
 353             if toolarg == "%loadPolly":
 354                 if not llvm_polly_link_into_tools:
 355                     newtool += [
 356                         "-load",
 357                         os.path.join(polly_lib_dir, "LLVMPolly" + shlibext),
 358                     ]
 359                 newtool.append("-polly-process-unprofitable")
 360                 newtool.append("-polly-remarks-minimal")
 361             elif toolarg == "2>&1":
 362                 optstderr = subprocess.STDOUT
 363             else:
 364                 newtool.append(toolarg)
 365         tool = newtool
 366
 367         inpfile = None
 368         i = 1
 369         while i < len(tool):
 370             if tool[i] == "<":
 371                 inpfile = tool[i + 1]
 372                 del tool[i : i + 2]
 373                 continue
 374             i += 1
 375         if inpfile:
 376             with open(inpfile) as inp:
 377                 retlines = subprocess.check_output(
 378                     tool, universal_newlines=True, stdin=inp, stderr=optstderr
 379                 )
 380         else:
 381             retlines = subprocess.check_output(
 382                 tool, universal_newlines=True, stderr=optstderr
 383             )
 384         retlines = [line.replace("\t", "    ") for line in retlines.splitlines()]
 385         check_include = []
 386         for checkme in known.check_include + known.check_label_include:
 387             parts = checkme.split("=")
 388             if len(parts) == 2:
 389                 if parts[0] == check_prefix:
 390                     check_include.append(parts[1])
 391             else:
 392                 check_include.append(checkme)
 393
 394         if check_include:
 395             filtered_retlines = []
 396             classified_retlines = []
 397             lastmatch = None
 398             for line, kind in (
 399                 (line, class1.union(class2))
 400                 for line, class1, class2 in zip(
 401                     retlines, classyfier1(retlines), classyfier2(retlines)
 402                 )
 403             ):
 404                 match = kind.intersection(check_include)
 405                 if match:
 406                     if lastmatch != match:
 407                         filtered_retlines.append("")
 408                         classified_retlines.append({"Separator"})
 409                     filtered_retlines.append(line)
 410                     classified_retlines.append(kind)
 411                 lastmatch = match
 412
 413             retlines = filtered_retlines
 414         else:
 415             classified_retlines = (set() for line in retlines)
 416
 417         rtrim_emptylines(retlines)
 418         ltrim_emptylines(retlines, classified_retlines)
 419         retlines = [
 420             replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines
 421         ]
 422         indent = common_indent(retlines)
 423         retlines = [line[indent:] for line in retlines]
 424         checklines = []
 425         previous_was_empty = True
 426         for line, kind in zip(retlines, classified_retlines):
 427             if line:
 428                 if known.check_style == "CHECK" and known.check_label_include:
 429                     if not kind.isdisjoint(known.check_label_include):
 430                         checklines.append("; " + check_prefix + "-LABEL: " + line)
 431                     else:
 432                         checklines.append("; " + check_prefix + ":       " + line)
 433                 elif known.check_style == "CHECK":
 434                     checklines.append("; " + check_prefix + ": " + line)
 435                 elif known.check_label_include and known.check_label_include:
 436                     if not kind.isdisjoint(known.check_label_include):
 437                         checklines.append("; " + check_prefix + "-LABEL: " + line)
 438                     elif previous_was_empty:
 439                         checklines.append("; " + check_prefix + ":       " + line)
 440                     else:
 441                         checklines.append("; " + check_prefix + "-NEXT:  " + line)
 442                 else:
 443                     if previous_was_empty:
 444                         checklines.append("; " + check_prefix + ":      " + line)
 445                     else:
 446                         checklines.append("; " + check_prefix + "-NEXT: " + line)
 447                 previous_was_empty = False
 448             else:
 449                 if not "Separator" in kind or known.check_part_newline:
 450                     checklines.append(";")
 451                 previous_was_empty = True
 452         allchecklines.append(checklines)
 453
 454     if not checkprefixes:
 455         return
 456
 457     checkre = re.compile(
 458         r"^\s*\;\s*("
 459         + "|".join([re.escape(s) for s in checkprefixes])
 460         + r")(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:"
 461     )
 462     firstcheckline = None
 463     firstnoncommentline = None
 464     headerlines = []
 465     newlines = []
 466     uptonowlines = []
 467     emptylines = []
 468     lastwascheck = False
 469     for line in oldlines:
 470         if checkre.match(line):
 471             if firstcheckline is None:
 472                 firstcheckline = len(newlines) + len(emptylines)
 473             if not lastwascheck:
 474                 uptonowlines += emptylines
 475             emptylines = []
 476             lastwascheck = True
 477         elif emptyline.fullmatch(line):
 478             emptylines.append(line)
 479         else:
 480             newlines += uptonowlines
 481             newlines += emptylines
 482             newlines.append(line)
 483             emptylines = []
 484             uptonowlines = []
 485             lastwascheck = False
 486
 487     for i, line in enumerate(newlines):
 488         if not commentline.fullmatch(line):
 489             firstnoncommentline = i
 490             break
 491
 492     with open(outfile, "w", newline="") as file:
 493
 494         def writelines(lines):
 495             for line in lines:
 496                 file.write(line)
 497                 file.write("\n")
 498
 499         if firstcheckline is not None and known.check_position == "autodetect":
 500             writelines(newlines[:firstcheckline])
 501             writelines(uptonowlines)
 502             for i, checklines in enumerate(allchecklines):
 503                 if i != 0:
 504                     file.write("\n")
 505                 writelines(checklines)
 506             writelines(newlines[firstcheckline:])
 507             writelines(emptylines)
 508         elif (
 509             firstnoncommentline is not None and known.check_position == "before-content"
 510         ):
 511             headerlines = newlines[:firstnoncommentline]
 512             rtrim_emptylines(headerlines)
 513             contentlines = newlines[firstnoncommentline:]
 514             ltrim_emptylines(contentlines)
 515
 516             writelines(headerlines)
 517             for checklines in allchecklines:
 518                 file.write("\n")
 519                 writelines(checklines)
 520             file.write("\n")
 521             writelines(contentlines)
 522             writelines(uptonowlines)
 523             writelines(emptylines)
 524         else:
 525             writelines(newlines)
 526             rtrim_emptylines(newlines)
 527             for checklines in allchecklines:
 528                 file.write("\n\n")
 529                 writelines(checklines)
 530
 531
 532 if __name__ == "__main__":
 533     main()