flang/examples/FlangOmpReport/yaml_summarizer.py

   1 """YAML Summariser
   2
   3 The flang plugin ``flang-omp-report`` takes one fortran
   4 file in and returns a YAML report file of the input file.
   5 This becomes an issue when you want to analyse an entire project
   6 into one final report.
   7 The purpose of this Python script is to generate a final YAML
   8 summary from all of the files generated by ``flang-omp-report``.
   9
  10 Currently, it requires ``ruamel.yaml``,
  11 which can be installed with:
  12
  13     ``pip3 install ruamel.yaml``
  14
  15 By default it scans the directory it is ran in
  16 for any YAML files and outputs a summary to
  17 stdout. It can be ran as:
  18
  19     ``python3 yaml_summarizer.py``
  20
  21 Parameters:
  22
  23     -d   --directory   Specify which directory to scan. Multiple directories can be searched by
  24                        providing a semicolon seperated list of directories.
  25
  26     -l   --log         Combine all yaml files into one log (instead of generating a summary)
  27
  28     -o   --output      Specify a directory in which to save the summary file
  29
  30     -r   --recursive   Recursively search directory for all yaml files
  31
  32 Examples:
  33
  34     ``python3 yaml_summarizer.py -d ~/llvm-project/build/ -r``
  35
  36     ``python3 yaml_summarizer.py -d "~/llvm-project/build/;~/llvm-project/flang/test/Examples"``
  37
  38     ``python3 yaml_summarizer.py -l -o ~/examples/report.yaml``
  39
  40 Pseudo-examples:
  41
  42     Summary:
  43
  44     $ python3 yaml_summarizer.py file_1.yaml file_2.yaml
  45     <Unique OMP constructs with there grouped clauses from file_1.yaml and file_2.yaml>
  46
  47     Construcsts are in the form:
  48     - construct: someOMPconstruct
  49     count: 8
  50     clauses:
  51     - clause: clauseOne
  52         count: 4
  53     - clause: ClauseTwo
  54         count: 2
  55
  56     Log:
  57
  58     $ python3 yaml_summarizer.py -l file_1.yaml file_2.yaml
  59     file_1.yaml
  60     <OMP clauses and constructs from file_1.yaml>
  61     file_2.yaml
  62     <OMP clauses and constructs from file_2.yaml>
  63
  64     Constructs are in the form:
  65     - construct: someOMPConstruct
  66     line: 12
  67     clauses:
  68     - clause: clauseOne
  69         details: 'someDetailForClause'
  70 """
  71
  72 import sys
  73 import glob
  74 import argparse
  75 from pathlib import Path
  76 from os.path import isdir
  77
  78 from ruamel.yaml import YAML
  79
  80 def find_yaml_files(search_directory: Path, search_pattern: str):
  81     """
  82     Find all '.yaml' files and returns an iglob iterator to them.
  83
  84     Keyword arguments:
  85     search_pattern -- Search pattern for 'iglob' to use for finding '.yaml' files.
  86                       If this is set to 'None', then it will default to just searching
  87                       for all '.yaml' files in the current directory.
  88     """
  89     # @TODO: Currently *all* yaml files are read - regardless of whether they have
  90     # been generated with  'flang-omp-report' or not. This might result in the script
  91     # reading files that it should ignore.
  92     if search_directory:
  93         return glob.iglob(str(search_directory.joinpath(search_pattern)), recursive=True)
  94
  95     return glob.iglob(str("/" + search_pattern), recursive=True)
  96
  97 def process_log(data, result: list):
  98     """
  99     Process the data input as a 'log' to the result array. This esssentially just
 100     stitches together all of the input '.yaml' files into one result.
 101
 102     Keyword arguments:
 103     data -- Data from yaml.load() for a yaml file. So the type can be 'Any'.
 104     result -- Array to add the processed data to.
 105     """
 106     for datum in data:
 107         items = result.get(datum['file'], [])
 108         items.append({"construct" : datum['construct'],
 109                         "line" : datum['line'],
 110                         "clauses" : datum['clauses']})
 111         result[datum['file']] = items
 112
 113 def add_clause(datum, construct):
 114     """
 115     Add clauses to the construct if they're missing
 116     Otherwise increment their count by one.
 117
 118     Keyword arguments:
 119     datum -- Data construct containing clauses to check.
 120     construct -- Construct to add or increment clause count.
 121     """
 122     to_check = [i['clause'] for i in construct['clauses']]
 123     to_add = [i['clause'] for i in datum['clauses']]
 124     clauses = construct["clauses"]
 125     for item in to_add:
 126         if item in to_check:
 127             for clause in clauses:
 128                 if clause["clause"] == item:
 129                     clause["count"] += 1
 130         else:
 131             clauses.append({"clause" : item,
 132                             "count" : 1})
 133
 134 def process_summary(data, result: dict):
 135     """
 136     Process the data input as a 'summary' to the 'result' dictionary.
 137
 138     Keyword arguments:
 139     data -- Data from yaml.load() for a yaml file. So the type can be 'Any'.
 140     result -- Dictionary to add the processed data to.
 141     """
 142     for datum in data:
 143         construct = next((item for item in result
 144                             if item["construct"] == datum["construct"]), None)
 145         clauses = []
 146         # Add the construct and clauses to the summary if
 147         # they haven't been seen before
 148         if not construct:
 149             for i in datum['clauses']:
 150                 clauses.append({"clause" : i['clause'],
 151                                 "count"    : 1})
 152             result.append({"construct" : datum['construct'],
 153                             "count" : 1,
 154                             "clauses" : clauses})
 155         else:
 156             construct["count"] += 1
 157
 158             add_clause(datum, construct)
 159
 160 def clean_summary(result):
 161     """ Cleans the result after processing the yaml files with summary format."""
 162     # Remove all "clauses" that are empty to keep things compact
 163     for construct in result:
 164         if construct["clauses"] == []:
 165             construct.pop("clauses")
 166
 167 def clean_log(result):
 168     """ Cleans the result after processing the yaml files with log format."""
 169     for constructs in result.values():
 170         for construct in constructs:
 171             if construct["clauses"] == []:
 172                 construct.pop("clauses")
 173
 174 def output_result(yaml: YAML, result, output_file: Path):
 175     """
 176     Outputs result to either 'stdout' or to a output file.
 177
 178     Keyword arguments:
 179     result -- Format result to output.
 180     output_file -- File to output result to. If this is 'None' then result will be
 181                    outputted to 'stdout'.
 182     """
 183     if output_file:
 184         with open(output_file, 'w+', encoding='utf-8') as file:
 185             if output_file.suffix == ".yaml":
 186                 yaml.dump(result, file)
 187             else:
 188                 file.write(result)
 189     else:
 190         yaml.dump(result, sys.stdout)
 191
 192 def process_yaml(search_directories: list, search_pattern: str,
 193                  result_format: str, output_file: Path):
 194     """
 195     Reads each yaml file, calls the appropiate format function for
 196     the file and then ouputs the result to either 'stdout' or to an output file.
 197
 198     Keyword arguments:
 199     search_directories -- List of directory paths to search for '.yaml' files in.
 200     search_pattern -- String pattern formatted for use with glob.iglob to find all
 201                       '.yaml' files.
 202     result_format -- String representing output format. Current supported strings are: 'log'.
 203     output_file -- Path to output file (If value is None, then default to outputting to 'stdout').
 204     """
 205     if result_format == "log":
 206         result = {}
 207         action = process_log
 208         clean_report = clean_log
 209     else:
 210         result = []
 211         action = process_summary
 212         clean_report = clean_summary
 213
 214     yaml = YAML()
 215
 216     for search_directory in search_directories:
 217         for file in find_yaml_files(search_directory, search_pattern):
 218             with open(file, "r", encoding='utf-8') as yaml_file:
 219                 data = yaml.load(yaml_file)
 220                 action(data, result)
 221
 222     if clean_report is not None:
 223         clean_report(result)
 224
 225     output_result(yaml, result, output_file)
 226
 227 def create_arg_parser():
 228     """ Create and return a argparse.ArgumentParser modified for script. """
 229     parser = argparse.ArgumentParser()
 230     parser.add_argument("-d", "--directory", help="Specify a directory to scan",
 231                         dest="dir", type=str)
 232     parser.add_argument("-o", "--output", help="Writes to a file instead of\
 233                                                 stdout", dest="output", type=str)
 234     parser.add_argument("-r", "--recursive", help="Recursive search for .yaml files",
 235                         dest="recursive", type=bool, nargs='?', const=True, default=False)
 236
 237     exclusive_parser = parser.add_mutually_exclusive_group()
 238     exclusive_parser.add_argument("-l", "--log", help="Modifies report format: "
 239                                   "Combines the log '.yaml' files into one file.",
 240                                   action='store_true', dest='log')
 241     return parser
 242
 243 def parse_arguments():
 244     """ Parses arguments given to script and returns a tuple of processed arguments. """
 245     parser = create_arg_parser()
 246     args = parser.parse_args()
 247
 248     if args.dir:
 249         search_directory = [Path(path) for path in args.dir.split(";")]
 250     else:
 251         search_directory = [Path.cwd()]
 252
 253     if args.recursive:
 254         search_pattern = "**/*.yaml"
 255     else:
 256         search_pattern = "*.yaml"
 257
 258     if args.log:
 259         result_format = "log"
 260     else:
 261         result_format = "summary"
 262
 263     if args.output:
 264         if isdir(args.output):
 265             output_file = Path(args.output).joinpath("summary.yaml")
 266         elif isdir(Path(args.output).resolve().parent):
 267             output_file = Path(args.output)
 268     else:
 269         output_file = None
 270
 271     return (search_directory, search_pattern, result_format, output_file)
 272
 273 def main():
 274     """ Main function of script. """
 275     (search_directory, search_pattern, result_format, output_file) = parse_arguments()
 276
 277     process_yaml(search_directory, search_pattern, result_format, output_file)
 278
 279     return 0
 280
 281 if __name__ == "__main__":
 282     sys.exit(main())