bolt/utils/nfc-stat-parser.py

   1 #!/usr/bin/env python3
   2 import argparse
   3 import csv
   4 import re
   5 import sys
   6 import os
   7 from statistics import geometric_mean
   8
   9 TIMING_LOG_RE = re.compile(r"(.*)/(.*).tmp(.*)")
  10
  11
  12 def main():
  13     parser = argparse.ArgumentParser(
  14         description="BOLT NFC stat parser",
  15         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
  16     )
  17     parser.add_argument(
  18         "input", nargs="+", help="timing.log files produced by llvm-bolt-wrapper"
  19     )
  20     parser.add_argument(
  21         "--check_longer_than",
  22         default=2,
  23         type=float,
  24         help="Only warn on tests longer than X seconds for at least one side",
  25     )
  26     parser.add_argument(
  27         "--threshold_single",
  28         default=10,
  29         type=float,
  30         help="Threshold for a single test result swing, abs percent",
  31     ),
  32     parser.add_argument(
  33         "--threshold_agg",
  34         default=5,
  35         type=float,
  36         help="Threshold for geomean test results swing, abs percent",
  37     ),
  38     parser.add_argument("--verbose", "-v", action="store_true")
  39     args = parser.parse_args()
  40
  41     def fmt_delta(value, exc_threshold, above_bound=True):
  42         formatted_value = format(value, "+.2%")
  43         if not above_bound:
  44             formatted_value += "?"
  45         elif exc_threshold and sys.stdout.isatty():  # terminal supports colors
  46             return f"\033[1m{formatted_value}\033[0m"
  47         return formatted_value
  48
  49     # Ratios for geomean computation
  50     time_ratios = []
  51     mem_ratios = []
  52     # Whether any test exceeds the single test threshold (mem or time)
  53     threshold_single = False
  54     # Whether geomean exceeds aggregate test threshold (mem or time)
  55     threshold_agg = False
  56
  57     if args.verbose:
  58         print(f"# Individual test threshold: +-{args.threshold_single}%")
  59         print(f"# Aggregate (geomean) test threshold: +-{args.threshold_agg}%")
  60         print(
  61             f"# Checking time swings for tests with runtime >"
  62             f"{args.check_longer_than}s - otherwise marked as ?"
  63         )
  64         print("Test/binary BOLT_wall_time BOLT_max_rss")
  65
  66     for input_file in args.input:
  67         input_dir = os.path.dirname(input_file)
  68         with open(input_file) as timing_file:
  69             timing_reader = csv.reader(timing_file, delimiter=";")
  70             for row in timing_reader:
  71                 test_name = row[0]
  72                 m = TIMING_LOG_RE.match(row[0])
  73                 if m:
  74                     test_name = f"{input_dir}/{m.groups()[1]}/{m.groups()[2]}"
  75                 else:
  76                     # Prepend input dir to unparsed test name
  77                     test_name = input_dir + "#" + test_name
  78                 time_a, time_b = float(row[1]), float(row[3])
  79                 mem_a, mem_b = int(row[2]), int(row[4])
  80                 # Check if time is above bound for at least one side
  81                 time_above_bound = any(
  82                     [x > args.check_longer_than for x in [time_a, time_b]]
  83                 )
  84                 # Compute B/A ratios (for % delta and geomean)
  85                 time_ratio = time_b / time_a if time_a else float('nan')
  86                 mem_ratio = mem_b / mem_a if mem_a else float('nan')
  87                 # Keep ratios for geomean
  88                 if time_above_bound and time_ratio > 0:  # must be >0 for gmean
  89                     time_ratios += [time_ratio]
  90                 mem_ratios += [mem_ratio]
  91                 # Deltas: (B/A)-1 = (B-A)/A
  92                 time_delta = time_ratio - 1
  93                 mem_delta = mem_ratio - 1
  94                 # Check individual test results vs single test threshold
  95                 time_exc = (
  96                     100.0 * abs(time_delta) > args.threshold_single and time_above_bound
  97                 )
  98                 mem_exc = 100.0 * abs(mem_delta) > args.threshold_single
  99                 if time_exc or mem_exc:
 100                     threshold_single = True
 101                 # Print deltas with formatting in verbose mode
 102                 if args.verbose or time_exc or mem_exc:
 103                     print(
 104                         test_name,
 105                         fmt_delta(time_delta, time_exc, time_above_bound),
 106                         fmt_delta(mem_delta, mem_exc),
 107                     )
 108
 109     time_gmean_delta = geometric_mean(time_ratios) - 1
 110     mem_gmean_delta = geometric_mean(mem_ratios) - 1
 111     time_agg_threshold = 100.0 * abs(time_gmean_delta) > args.threshold_agg
 112     mem_agg_threshold = 100.0 * abs(mem_gmean_delta) > args.threshold_agg
 113     if time_agg_threshold or mem_agg_threshold:
 114         threshold_agg = True
 115     if time_agg_threshold or mem_agg_threshold or args.verbose:
 116         print(
 117             "Geomean",
 118             fmt_delta(time_gmean_delta, time_agg_threshold),
 119             fmt_delta(mem_gmean_delta, mem_agg_threshold),
 120         )
 121     exit(threshold_single or threshold_agg)
 122
 123
 124 if __name__ == "__main__":
 125     main()