clang/test/AST/gen_ast_dump_json_test.py

   1 #!/usr/bin/env python3
   2
   3 from __future__ import print_function
   4 from collections import OrderedDict
   5 from shutil import copyfile
   6 import argparse
   7 import json
   8 import os
   9 import re
  10 import subprocess
  11 import sys
  12 import tempfile
  13
  14
  15 def normalize(dict_var):
  16     for k, v in dict_var.items():
  17         if isinstance(v, OrderedDict):
  18             normalize(v)
  19         elif isinstance(v, list):
  20             for e in v:
  21                 if isinstance(e, OrderedDict):
  22                     normalize(e)
  23         elif type(v) is str:
  24             if v != "0x0" and re.match(r"0x[0-9A-Fa-f]+", v):
  25                 dict_var[k] = "0x{{.*}}"
  26             elif os.path.isfile(v):
  27                 dict_var[k] = "{{.*}}"
  28             else:
  29                 splits = v.split(" ")
  30                 out_splits = []
  31                 for split in splits:
  32                     inner_splits = split.rsplit(":", 2)
  33                     if os.path.isfile(inner_splits[0]):
  34                         out_splits.append(
  35                             "{{.*}}:%s:%s" % (inner_splits[1], inner_splits[2])
  36                         )
  37                         continue
  38                     out_splits.append(split)
  39
  40                 dict_var[k] = " ".join(out_splits)
  41
  42
  43 def filter_json(dict_var, filters, out):
  44     for k, v in dict_var.items():
  45         if type(v) is str:
  46             if v in filters:
  47                 out.append(dict_var)
  48                 break
  49         elif isinstance(v, OrderedDict):
  50             filter_json(v, filters, out)
  51         elif isinstance(v, list):
  52             for e in v:
  53                 if isinstance(e, OrderedDict):
  54                     filter_json(e, filters, out)
  55
  56
  57 def default_clang_path():
  58     guessed_clang = os.path.join(os.path.dirname(__file__), "clang")
  59     if os.path.isfile(guessed_clang):
  60         return guessed_clang
  61     return None
  62
  63
  64 def main():
  65     parser = argparse.ArgumentParser()
  66     parser.add_argument(
  67         "--clang",
  68         help="The clang binary (could be a relative or absolute path)",
  69         action="store",
  70         default=default_clang_path(),
  71     )
  72     parser.add_argument(
  73         "--source",
  74         help="the source file(s). Without --update, the command used to generate the JSON "
  75         "will be of the format <clang> -cc1 -ast-dump=json <opts> <source>",
  76         action="store",
  77         nargs=argparse.ONE_OR_MORE,
  78         required=True,
  79     )
  80     parser.add_argument(
  81         "--filters",
  82         help="comma separated list of AST filters. Ex: --filters=TypedefDecl,BuiltinType",
  83         action="store",
  84         default="",
  85     )
  86     update_or_generate_group = parser.add_mutually_exclusive_group()
  87     update_or_generate_group.add_argument(
  88         "--update", help="Update the file in-place", action="store_true"
  89     )
  90     update_or_generate_group.add_argument(
  91         "--opts", help="other options", action="store", default="", type=str
  92     )
  93     parser.add_argument(
  94         "--update-manual",
  95         help="When using --update, also update files that do not have the "
  96         "autogenerated disclaimer",
  97         action="store_true",
  98     )
  99     args = parser.parse_args()
 100
 101     if not args.source:
 102         sys.exit("Specify the source file to give to clang.")
 103
 104     clang_binary = os.path.abspath(args.clang)
 105     if not os.path.isfile(clang_binary):
 106         sys.exit("clang binary specified not present.")
 107
 108     for src in args.source:
 109         process_file(
 110             src,
 111             clang_binary,
 112             cmdline_filters=args.filters,
 113             cmdline_opts=args.opts,
 114             do_update=args.update,
 115             force_update=args.update_manual,
 116         )
 117
 118
 119 def process_file(
 120     source_file, clang_binary, cmdline_filters, cmdline_opts, do_update, force_update
 121 ):
 122     note_firstline = (
 123         "// NOTE: CHECK lines have been autogenerated by " "gen_ast_dump_json_test.py"
 124     )
 125     filters_line_prefix = "// using --filters="
 126     note = note_firstline
 127
 128     cmd = [clang_binary, "-cc1"]
 129     if do_update:
 130         # When updating the first line of the test must be a RUN: line
 131         with open(source_file, "r") as srcf:
 132             first_line = srcf.readline()
 133             found_autogenerated_line = False
 134             filters_line = None
 135             for i, line in enumerate(srcf.readlines()):
 136                 if found_autogenerated_line:
 137                     # print("Filters line: '", line.rstrip(), "'", sep="")
 138                     if line.startswith(filters_line_prefix):
 139                         filters_line = line[len(filters_line_prefix) :].rstrip()
 140                     break
 141                 if line.startswith(note_firstline):
 142                     found_autogenerated_line = True
 143                     # print("Found autogenerated disclaimer at line", i + 1)
 144         if not found_autogenerated_line and not force_update:
 145             print(
 146                 "Not updating",
 147                 source_file,
 148                 "since it is not autogenerated.",
 149                 file=sys.stderr,
 150             )
 151             return
 152         if not cmdline_filters and filters_line:
 153             cmdline_filters = filters_line
 154             print("Inferred filters as '" + cmdline_filters + "'")
 155
 156         if "RUN: %clang_cc1 " not in first_line:
 157             sys.exit(
 158                 "When using --update the first line of the input file must contain RUN: %clang_cc1"
 159             )
 160         clang_start = first_line.find("%clang_cc1") + len("%clang_cc1")
 161         file_check_idx = first_line.rfind("| FileCheck")
 162         if file_check_idx:
 163             dump_cmd = first_line[clang_start:file_check_idx]
 164         else:
 165             dump_cmd = first_line[clang_start:]
 166         print("Inferred run arguments as '", dump_cmd, "'", sep="")
 167         options = dump_cmd.split()
 168         if "-ast-dump=json" not in options:
 169             sys.exit("ERROR: RUN: line does not contain -ast-dump=json")
 170         if "%s" not in options:
 171             sys.exit("ERROR: RUN: line does not contain %s")
 172         options.remove("%s")
 173     else:
 174         options = cmdline_opts.split()
 175         options.append("-ast-dump=json")
 176     cmd.extend(options)
 177     using_ast_dump_filter = any("ast-dump-filter" in arg for arg in cmd)
 178     cmd.append(source_file)
 179     print("Will run", cmd)
 180     filters = set()
 181     if cmdline_filters:
 182         note += "\n" + filters_line_prefix + cmdline_filters
 183         filters = set(cmdline_filters.split(","))
 184     print("Will use the following filters:", filters)
 185
 186     try:
 187         json_str = subprocess.check_output(cmd).decode()
 188     except Exception as ex:
 189         print("The clang command failed with %s" % ex)
 190         return -1
 191
 192     out_asts = []
 193     if using_ast_dump_filter:
 194         # If we're using a filter, then we might have multiple JSON objects
 195         # in the output. To parse each out, we use a manual JSONDecoder in
 196         # "raw" mode and update our location in the string based on where the
 197         # last document ended.
 198         decoder = json.JSONDecoder(object_hook=OrderedDict)
 199         doc_start = 0
 200         prev_end = 0
 201         while True:
 202             try:
 203                 prev_end = doc_start
 204                 (j, doc_start) = decoder.raw_decode(json_str[doc_start:])
 205                 doc_start += prev_end + 1
 206                 normalize(j)
 207                 out_asts.append(j)
 208             except:
 209                 break
 210     else:
 211         j = json.loads(json_str, object_pairs_hook=OrderedDict)
 212         normalize(j)
 213
 214         if len(filters) == 0:
 215             out_asts.append(j)
 216         else:
 217             filter_json(j, filters, out_asts)
 218
 219     with tempfile.NamedTemporaryFile("w", delete=False) as f:
 220         with open(source_file, "r") as srcf:
 221             for line in srcf.readlines():
 222                 # copy up to the note:
 223                 if line.rstrip() == note_firstline:
 224                     break
 225                 f.write(line)
 226         f.write(note + "\n")
 227         for out_ast in out_asts:
 228             append_str = json.dumps(out_ast, indent=1, ensure_ascii=False)
 229             out_str = "\n\n"
 230             out_str += "// CHECK-NOT: {{^}}Dumping\n"
 231             index = 0
 232             for append_line in append_str.splitlines()[2:]:
 233                 if index == 0:
 234                     out_str += "// CHECK: %s\n" % (append_line.rstrip())
 235                     index += 1
 236                 else:
 237                     out_str += "// CHECK-NEXT: %s\n" % (append_line.rstrip())
 238
 239             f.write(out_str)
 240         f.flush()
 241         f.close()
 242         if do_update:
 243             print("Updating json appended source file to %s." % source_file)
 244             copyfile(f.name, source_file)
 245         else:
 246             partition = source_file.rpartition(".")
 247             dest_path = "%s-json%s%s" % (partition[0], partition[1], partition[2])
 248             print("Writing json appended source file to %s." % dest_path)
 249             copyfile(f.name, dest_path)
 250         os.remove(f.name)
 251     return 0
 252
 253
 254 if __name__ == "__main__":
 255     main()