polly/test/update_check.py

   1 #!/usr/bin/env python3
   2 # -*- coding: UTF-8 -*-
   3
   4 # Polly/LLVM update_check.py
   5 # Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command.
   6
   7 import argparse
   8 import os
   9 import subprocess
  10 import shlex
  11 import re
  12
  13
  14 polly_src_dir = """@POLLY_SOURCE_DIR@"""
  15 polly_lib_dir = """@POLLY_LIB_DIR@"""
  16 shlibext = """@LLVM_SHLIBEXT@"""
  17 llvm_tools_dir = """@LLVM_TOOLS_DIR@"""
  18 llvm_polly_link_into_tools = not """@LLVM_POLLY_LINK_INTO_TOOLS@""".lower() in {
  19     "",
  20     "0",
  21     "n",
  22     "no",
  23     "off",
  24     "false",
  25     "notfound",
  26     "llvm_polly_link_into_tools-notfound",
  27 }
  28
  29 runre = re.compile(r"\s*\;\s*RUN\s*\:(?P<tool>.*)")
  30 filecheckre = re.compile(r"\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)")
  31 emptyline = re.compile(r"\s*(\;\s*)?")
  32 commentline = re.compile(r"\s*(\;.*)?")
  33
  34
  35 def ltrim_emptylines(lines, meta=None):
  36     while len(lines) and emptyline.fullmatch(lines[0]):
  37         del lines[0]
  38         if meta is not None:
  39             del meta[0]
  40
  41
  42 def rtrim_emptylines(lines):
  43     while len(lines) and emptyline.fullmatch(lines[-1]):
  44         del lines[-1]
  45
  46
  47 def trim_emptylines(lines):
  48     ltrim_emptylines(lines)
  49     rtrim_emptylines(lines)
  50
  51
  52 def complete_exename(path, filename):
  53     complpath = os.path.join(path, filename)
  54     if os.path.isfile(complpath):
  55         return complpath
  56     elif os.path.isfile(complpath + ".exe"):
  57         return complpath + ".exe"
  58     return filename
  59
  60
  61 def indention(line):
  62     for i, c in enumerate(line):
  63         if c != " " and c != "\t":
  64             return i
  65     return None
  66
  67
  68 def common_indent(lines):
  69     indentions = (indention(line) for line in lines)
  70     indentions = (indent for indent in indentions if indent is not None)
  71     return min(indentions, default=0)
  72
  73
  74 funcre = re.compile(r"^    Function: \S*$")
  75 regionre = re.compile(r"^    Region: \S*$")
  76 depthre = re.compile(r"^    Max Loop Depth: .*")
  77 paramre = re.compile(r"    [0-9a-z-A-Z_]+\: .*")
  78
  79
  80 def classyfier1(lines):
  81     i = iter(lines)
  82     line = i.__next__()
  83     while True:
  84         if line.startswith(
  85             "Printing analysis 'Polly - Calculate dependences' for region: "
  86         ):
  87             yield {"PrintingDependenceInfo"}
  88         elif line.startswith("remark: "):
  89             yield {"Remark"}
  90         elif funcre.fullmatch(line):
  91             yield {"Function"}
  92         elif regionre.fullmatch(line):
  93             yield {"Region"}
  94         elif depthre.fullmatch(line):
  95             yield {"MaxLoopDepth"}
  96         elif line == "    Invariant Accesses: {":
  97             while True:
  98                 yield {"InvariantAccesses"}
  99                 if line == "    }":
 100                     break
 101                 line = i.__next__()
 102         elif line == "    Context:":
 103             yield {"Context"}
 104             line = i.__next__()
 105             yield {"Context"}
 106         elif line == "    Assumed Context:":
 107             yield {"AssumedContext"}
 108             line = i.__next__()
 109             yield {"AssumedContext"}
 110         elif line == "    Invalid Context:":
 111             yield {"InvalidContext"}
 112             line = i.__next__()
 113             yield {"InvalidContext"}
 114         elif line == "    Boundary Context:":
 115             yield {"BoundaryContext"}
 116             line = i.__next__()
 117             yield {"BoundaryContext"}
 118             line = i.__next__()
 119             while paramre.fullmatch(line):
 120                 yield {"Param"}
 121                 line = i.__next__()
 122             continue
 123         elif line == "    Arrays {":
 124             while True:
 125                 yield {"Arrays"}
 126                 if line == "    }":
 127                     break
 128                 line = i.__next__()
 129         elif line == "    Arrays (Bounds as pw_affs) {":
 130             while True:
 131                 yield {"PwAffArrays"}
 132                 if line == "    }":
 133                     break
 134                 line = i.__next__()
 135         elif line.startswith("    Alias Groups ("):
 136             while True:
 137                 yield {"AliasGroups"}
 138                 line = i.__next__()
 139                 if not line.startswith("        "):
 140                     break
 141             continue
 142         elif line == "    Statements {":
 143             while True:
 144                 yield {"Statements"}
 145                 if line == "    }":
 146                     break
 147                 line = i.__next__()
 148         elif line == "    RAW dependences:":
 149             yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}
 150             line = i.__next__()
 151             while line.startswith("        "):
 152                 yield {"RAWDep", "BasicDep", "Dep", "DepInfo"}
 153                 line = i.__next__()
 154             continue
 155         elif line == "    WAR dependences:":
 156             yield {"WARDep", "BasicDep", "Dep", "DepInfo"}
 157             line = i.__next__()
 158             while line.startswith("        "):
 159                 yield {"WARDep", "BasicDep", "Dep", "DepInfo"}
 160                 line = i.__next__()
 161             continue
 162         elif line == "    WAW dependences:":
 163             yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}
 164             line = i.__next__()
 165             while line.startswith("        "):
 166                 yield {"WAWDep", "BasicDep", "Dep", "DepInfo"}
 167                 line = i.__next__()
 168             continue
 169         elif line == "    Reduction dependences:":
 170             yield {"RedDep", "Dep", "DepInfo"}
 171             line = i.__next__()
 172             while line.startswith("        "):
 173                 yield {"RedDep", "Dep", "DepInfo"}
 174                 line = i.__next__()
 175             continue
 176         elif line == "    Transitive closure of reduction dependences:":
 177             yield {"TransitiveClosureDep", "DepInfo"}
 178             line = i.__next__()
 179             while line.startswith("        "):
 180                 yield {"TransitiveClosureDep", "DepInfo"}
 181                 line = i.__next__()
 182             continue
 183         elif line.startswith("New access function '"):
 184             yield {"NewAccessFunction"}
 185         elif line == "Schedule before flattening {":
 186             while True:
 187                 yield {"ScheduleBeforeFlattening"}
 188                 if line == "}":
 189                     break
 190                 line = i.__next__()
 191         elif line == "Schedule after flattening {":
 192             while True:
 193                 yield {"ScheduleAfterFlattening"}
 194                 if line == "}":
 195                     break
 196                 line = i.__next__()
 197         else:
 198             yield set()
 199         line = i.__next__()
 200
 201
 202 def classyfier2(lines):
 203     i = iter(lines)
 204     line = i.__next__()
 205     while True:
 206         if funcre.fullmatch(line):
 207             while line.startswith("    "):
 208                 yield {"FunctionDetail"}
 209                 line = i.__next__()
 210             continue
 211         elif line.startswith(
 212             "Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: "
 213         ):
 214             yield {"PrintingIslAst"}
 215             line = i.__next__()
 216             while not line.startswith("Printing analysis"):
 217                 yield {"AstDetail"}
 218                 line = i.__next__()
 219             continue
 220         else:
 221             yield set()
 222         line = i.__next__()
 223
 224
 225 replrepl = {"{{": "{{[{][{]}}", "}}": "{{[}][}]}}", "[[": "{{\[\[}}", "]]": "{{\]\]}}"}
 226 replre = re.compile("|".join(re.escape(k) for k in replrepl.keys()))
 227
 228
 229 def main():
 230     parser = argparse.ArgumentParser(description="Update CHECK lines")
 231     parser.add_argument(
 232         "testfile", help="File to update (absolute or relative to --testdir)"
 233     )
 234     parser.add_argument(
 235         "--check-style",
 236         choices=["CHECK", "CHECK-NEXT"],
 237         default="CHECK-NEXT",
 238         help="What kind of checks lines to generate",
 239     )
 240     parser.add_argument(
 241         "--check-position",
 242         choices=["end", "before-content", "autodetect"],
 243         default="autodetect",
 244         help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there",
 245     )
 246     parser.add_argument(
 247         "--check-include",
 248         action="append",
 249         default=[],
 250         help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)",
 251     )
 252     parser.add_argument(
 253         "--check-label-include",
 254         action="append",
 255         default=[],
 256         help="Use CHECK-LABEL for these includes",
 257     )
 258     parser.add_argument(
 259         "--check-part-newline",
 260         action="store_true",
 261         help="Add empty line between different check parts",
 262     )
 263     parser.add_argument(
 264         "--prefix-only",
 265         action="append",
 266         default=None,
 267         help="Update only these prefixes (default: all)",
 268     )
 269     parser.add_argument("--bindir", help="Location of the opt program")
 270     parser.add_argument("--testdir", help="Root dir for unit tests")
 271     parser.add_argument(
 272         "--inplace", "-i", action="store_true", help="Replace input file"
 273     )
 274     parser.add_argument("--output", "-o", help="Write changed input to this file")
 275     known = parser.parse_args()
 276
 277     if not known.inplace and known.output is None:
 278         print("Must specify what to do with output (--output or --inplace)")
 279         exit(1)
 280     if known.inplace and known.output is not None:
 281         print("--inplace and --output are mutually exclusive")
 282         exit(1)
 283
 284     outfile = known.output
 285
 286     filecheckparser = argparse.ArgumentParser(add_help=False)
 287     filecheckparser.add_argument("-check-prefix", "--check-prefix", default="CHECK")
 288
 289     filename = known.testfile
 290     for dir in [".", known.testdir, os.path.join(polly_src_dir, "test"), polly_src_dir]:
 291         if not dir:
 292             continue
 293         testfilename = os.path.join(dir, filename)
 294         if os.path.isfile(testfilename):
 295             filename = testfilename
 296             break
 297
 298     if known.inplace:
 299         outfile = filename
 300
 301     allchecklines = []
 302     checkprefixes = []
 303
 304     with open(filename, "r") as file:
 305         oldlines = [line.rstrip("\r\n") for line in file.readlines()]
 306
 307     runlines = []
 308     for line in oldlines:
 309         m = runre.match(line)
 310         if m:
 311             runlines.append(m.group("tool"))
 312
 313     continuation = ""
 314     newrunlines = []
 315     for line in runlines:
 316         if line.endswith("\\"):
 317             continuation += line[:-2] + " "
 318         else:
 319             newrunlines.append(continuation + line)
 320             continuation = ""
 321     if continuation:
 322         newrunlines.append(continuation)
 323
 324     for line in newrunlines:
 325         m = filecheckre.match(line)
 326         if not m:
 327             continue
 328
 329         tool, filecheck = m.group("tool", "filecheck")
 330         filecheck = shlex.split(filecheck)
 331         tool = shlex.split(tool)
 332         if known.bindir is not None:
 333             tool[0] = complete_exename(known.bindir, tool[0])
 334         if os.path.isdir(llvm_tools_dir):
 335             tool[0] = complete_exename(llvm_tools_dir, tool[0])
 336         check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix
 337         if known.prefix_only is not None and not check_prefix in known.prefix_only:
 338             continue
 339         if check_prefix in checkprefixes:
 340             continue
 341         checkprefixes.append(check_prefix)
 342
 343         newtool = []
 344         optstderr = None
 345         for toolarg in tool:
 346             toolarg = toolarg.replace("%s", filename)
 347             toolarg = toolarg.replace("%S", os.path.dirname(filename))
 348             if toolarg == "%loadPolly":
 349                 if not llvm_polly_link_into_tools:
 350                     newtool += [
 351                         "-load",
 352                         os.path.join(polly_lib_dir, "LLVMPolly" + shlibext),
 353                     ]
 354                 newtool.append("-polly-process-unprofitable")
 355                 newtool.append("-polly-remarks-minimal")
 356             elif toolarg == "2>&1":
 357                 optstderr = subprocess.STDOUT
 358             else:
 359                 newtool.append(toolarg)
 360         tool = newtool
 361
 362         inpfile = None
 363         i = 1
 364         while i < len(tool):
 365             if tool[i] == "<":
 366                 inpfile = tool[i + 1]
 367                 del tool[i : i + 2]
 368                 continue
 369             i += 1
 370         if inpfile:
 371             with open(inpfile) as inp:
 372                 retlines = subprocess.check_output(
 373                     tool, universal_newlines=True, stdin=inp, stderr=optstderr
 374                 )
 375         else:
 376             retlines = subprocess.check_output(
 377                 tool, universal_newlines=True, stderr=optstderr
 378             )
 379         retlines = [line.replace("\t", "    ") for line in retlines.splitlines()]
 380         check_include = []
 381         for checkme in known.check_include + known.check_label_include:
 382             parts = checkme.split("=")
 383             if len(parts) == 2:
 384                 if parts[0] == check_prefix:
 385                     check_include.append(parts[1])
 386             else:
 387                 check_include.append(checkme)
 388
 389         if check_include:
 390             filtered_retlines = []
 391             classified_retlines = []
 392             lastmatch = None
 393             for line, kind in (
 394                 (line, class1.union(class2))
 395                 for line, class1, class2 in zip(
 396                     retlines, classyfier1(retlines), classyfier2(retlines)
 397                 )
 398             ):
 399                 match = kind.intersection(check_include)
 400                 if match:
 401                     if lastmatch != match:
 402                         filtered_retlines.append("")
 403                         classified_retlines.append({"Separator"})
 404                     filtered_retlines.append(line)
 405                     classified_retlines.append(kind)
 406                 lastmatch = match
 407
 408             retlines = filtered_retlines
 409         else:
 410             classified_retlines = (set() for line in retlines)
 411
 412         rtrim_emptylines(retlines)
 413         ltrim_emptylines(retlines, classified_retlines)
 414         retlines = [
 415             replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines
 416         ]
 417         indent = common_indent(retlines)
 418         retlines = [line[indent:] for line in retlines]
 419         checklines = []
 420         previous_was_empty = True
 421         for line, kind in zip(retlines, classified_retlines):
 422             if line:
 423                 if known.check_style == "CHECK" and known.check_label_include:
 424                     if not kind.isdisjoint(known.check_label_include):
 425                         checklines.append("; " + check_prefix + "-LABEL: " + line)
 426                     else:
 427                         checklines.append("; " + check_prefix + ":       " + line)
 428                 elif known.check_style == "CHECK":
 429                     checklines.append("; " + check_prefix + ": " + line)
 430                 elif known.check_label_include and known.check_label_include:
 431                     if not kind.isdisjoint(known.check_label_include):
 432                         checklines.append("; " + check_prefix + "-LABEL: " + line)
 433                     elif previous_was_empty:
 434                         checklines.append("; " + check_prefix + ":       " + line)
 435                     else:
 436                         checklines.append("; " + check_prefix + "-NEXT:  " + line)
 437                 else:
 438                     if previous_was_empty:
 439                         checklines.append("; " + check_prefix + ":      " + line)
 440                     else:
 441                         checklines.append("; " + check_prefix + "-NEXT: " + line)
 442                 previous_was_empty = False
 443             else:
 444                 if not "Separator" in kind or known.check_part_newline:
 445                     checklines.append(";")
 446                 previous_was_empty = True
 447         allchecklines.append(checklines)
 448
 449     if not checkprefixes:
 450         return
 451
 452     checkre = re.compile(
 453         r"^\s*\;\s*("
 454         + "|".join([re.escape(s) for s in checkprefixes])
 455         + ")(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:"
 456     )
 457     firstcheckline = None
 458     firstnoncommentline = None
 459     headerlines = []
 460     newlines = []
 461     uptonowlines = []
 462     emptylines = []
 463     lastwascheck = False
 464     for line in oldlines:
 465         if checkre.match(line):
 466             if firstcheckline is None:
 467                 firstcheckline = len(newlines) + len(emptylines)
 468             if not lastwascheck:
 469                 uptonowlines += emptylines
 470             emptylines = []
 471             lastwascheck = True
 472         elif emptyline.fullmatch(line):
 473             emptylines.append(line)
 474         else:
 475             newlines += uptonowlines
 476             newlines += emptylines
 477             newlines.append(line)
 478             emptylines = []
 479             uptonowlines = []
 480             lastwascheck = False
 481
 482     for i, line in enumerate(newlines):
 483         if not commentline.fullmatch(line):
 484             firstnoncommentline = i
 485             break
 486
 487     with open(outfile, "w", newline="") as file:
 488
 489         def writelines(lines):
 490             for line in lines:
 491                 file.write(line)
 492                 file.write("\n")
 493
 494         if firstcheckline is not None and known.check_position == "autodetect":
 495             writelines(newlines[:firstcheckline])
 496             writelines(uptonowlines)
 497             for i, checklines in enumerate(allchecklines):
 498                 if i != 0:
 499                     file.write("\n")
 500                 writelines(checklines)
 501             writelines(newlines[firstcheckline:])
 502             writelines(emptylines)
 503         elif (
 504             firstnoncommentline is not None and known.check_position == "before-content"
 505         ):
 506             headerlines = newlines[:firstnoncommentline]
 507             rtrim_emptylines(headerlines)
 508             contentlines = newlines[firstnoncommentline:]
 509             ltrim_emptylines(contentlines)
 510
 511             writelines(headerlines)
 512             for checklines in allchecklines:
 513                 file.write("\n")
 514                 writelines(checklines)
 515             file.write("\n")
 516             writelines(contentlines)
 517             writelines(uptonowlines)
 518             writelines(emptylines)
 519         else:
 520             writelines(newlines)
 521             rtrim_emptylines(newlines)
 522             for checklines in allchecklines:
 523                 file.write("\n\n")
 524                 writelines(checklines)
 525
 526
 527 if __name__ == "__main__":
 528     main()