utils/benchmark/tools/gbench/report.py

   1 """report.py - Utilities for reporting statistics about benchmark results
   2 """
   3 import os
   4 import re
   5 import copy
   6
   7 class BenchmarkColor(object):
   8     def __init__(self, name, code):
   9         self.name = name
  10         self.code = code
  11
  12     def __repr__(self):
  13         return '%s%r' % (self.__class__.__name__,
  14                          (self.name, self.code))
  15
  16     def __format__(self, format):
  17         return self.code
  18
  19 # Benchmark Colors Enumeration
  20 BC_NONE = BenchmarkColor('NONE', '')
  21 BC_MAGENTA = BenchmarkColor('MAGENTA', '\033[95m')
  22 BC_CYAN = BenchmarkColor('CYAN', '\033[96m')
  23 BC_OKBLUE = BenchmarkColor('OKBLUE', '\033[94m')
  24 BC_HEADER = BenchmarkColor('HEADER', '\033[92m')
  25 BC_WARNING = BenchmarkColor('WARNING', '\033[93m')
  26 BC_WHITE = BenchmarkColor('WHITE', '\033[97m')
  27 BC_FAIL = BenchmarkColor('FAIL', '\033[91m')
  28 BC_ENDC = BenchmarkColor('ENDC', '\033[0m')
  29 BC_BOLD = BenchmarkColor('BOLD', '\033[1m')
  30 BC_UNDERLINE = BenchmarkColor('UNDERLINE', '\033[4m')
  31
  32 def color_format(use_color, fmt_str, *args, **kwargs):
  33     """
  34     Return the result of 'fmt_str.format(*args, **kwargs)' after transforming
  35     'args' and 'kwargs' according to the value of 'use_color'. If 'use_color'
  36     is False then all color codes in 'args' and 'kwargs' are replaced with
  37     the empty string.
  38     """
  39     assert use_color is True or use_color is False
  40     if not use_color:
  41         args = [arg if not isinstance(arg, BenchmarkColor) else BC_NONE
  42                 for arg in args]
  43         kwargs = {key: arg if not isinstance(arg, BenchmarkColor) else BC_NONE
  44                   for key, arg in kwargs.items()}
  45     return fmt_str.format(*args, **kwargs)
  46
  47
  48 def find_longest_name(benchmark_list):
  49     """
  50     Return the length of the longest benchmark name in a given list of
  51     benchmark JSON objects
  52     """
  53     longest_name = 1
  54     for bc in benchmark_list:
  55         if len(bc['name']) > longest_name:
  56             longest_name = len(bc['name'])
  57     return longest_name
  58
  59
  60 def calculate_change(old_val, new_val):
  61     """
  62     Return a float representing the decimal change between old_val and new_val.
  63     """
  64     if old_val == 0 and new_val == 0:
  65         return 0.0
  66     if old_val == 0:
  67         return float(new_val - old_val) / (float(old_val + new_val) / 2)
  68     return float(new_val - old_val) / abs(old_val)
  69
  70
  71 def filter_benchmark(json_orig, family, replacement=""):
  72     """
  73     Apply a filter to the json, and only leave the 'family' of benchmarks.
  74     """
  75     regex = re.compile(family)
  76     filtered = {}
  77     filtered['benchmarks'] = []
  78     for be in json_orig['benchmarks']:
  79         if not regex.search(be['name']):
  80             continue
  81         filteredbench = copy.deepcopy(be) # Do NOT modify the old name!
  82         filteredbench['name'] = regex.sub(replacement, filteredbench['name'])
  83         filtered['benchmarks'].append(filteredbench)
  84     return filtered
  85
  86
  87 def generate_difference_report(json1, json2, use_color=True):
  88     """
  89     Calculate and report the difference between each test of two benchmarks
  90     runs specified as 'json1' and 'json2'.
  91     """
  92     first_col_width = find_longest_name(json1['benchmarks'])
  93     def find_test(name):
  94         for b in json2['benchmarks']:
  95             if b['name'] == name:
  96                 return b
  97         return None
  98     first_col_width = max(first_col_width, len('Benchmark'))
  99     first_line = "{:<{}s}Time             CPU      Time Old      Time New       CPU Old       CPU New".format(
 100         'Benchmark', 12 + first_col_width)
 101     output_strs = [first_line, '-' * len(first_line)]
 102
 103     gen = (bn for bn in json1['benchmarks'] if 'real_time' in bn and 'cpu_time' in bn)
 104     for bn in gen:
 105         other_bench = find_test(bn['name'])
 106         if not other_bench:
 107             continue
 108
 109         if bn['time_unit'] != other_bench['time_unit']:
 110             continue
 111
 112         def get_color(res):
 113             if res > 0.05:
 114                 return BC_FAIL
 115             elif res > -0.07:
 116                 return BC_WHITE
 117             else:
 118                 return BC_CYAN
 119         fmt_str = "{}{:<{}s}{endc}{}{:+16.4f}{endc}{}{:+16.4f}{endc}{:14.0f}{:14.0f}{endc}{:14.0f}{:14.0f}"
 120         tres = calculate_change(bn['real_time'], other_bench['real_time'])
 121         cpures = calculate_change(bn['cpu_time'], other_bench['cpu_time'])
 122         output_strs += [color_format(use_color, fmt_str,
 123             BC_HEADER, bn['name'], first_col_width,
 124             get_color(tres), tres, get_color(cpures), cpures,
 125             bn['real_time'], other_bench['real_time'],
 126             bn['cpu_time'], other_bench['cpu_time'],
 127             endc=BC_ENDC)]
 128     return output_strs
 129
 130 ###############################################################################
 131 # Unit tests
 132
 133 import unittest
 134
 135 class TestReportDifference(unittest.TestCase):
 136     def load_results(self):
 137         import json
 138         testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Inputs')
 139         testOutput1 = os.path.join(testInputs, 'test1_run1.json')
 140         testOutput2 = os.path.join(testInputs, 'test1_run2.json')
 141         with open(testOutput1, 'r') as f:
 142             json1 = json.load(f)
 143         with open(testOutput2, 'r') as f:
 144             json2 = json.load(f)
 145         return json1, json2
 146
 147     def test_basic(self):
 148         expect_lines = [
 149             ['BM_SameTimes', '+0.0000', '+0.0000', '10', '10', '10', '10'],
 150             ['BM_2xFaster', '-0.5000', '-0.5000', '50', '25', '50', '25'],
 151             ['BM_2xSlower', '+1.0000', '+1.0000', '50', '100', '50', '100'],
 152             ['BM_1PercentFaster', '-0.0100', '-0.0100', '100', '99', '100', '99'],
 153             ['BM_1PercentSlower', '+0.0100', '+0.0100', '100', '101', '100', '101'],
 154             ['BM_10PercentFaster', '-0.1000', '-0.1000', '100', '90', '100', '90'],
 155             ['BM_10PercentSlower', '+0.1000', '+0.1000', '100', '110', '100', '110'],
 156             ['BM_100xSlower', '+99.0000', '+99.0000', '100', '10000', '100', '10000'],
 157             ['BM_100xFaster', '-0.9900', '-0.9900', '10000', '100', '10000', '100'],
 158             ['BM_10PercentCPUToTime', '+0.1000', '-0.1000', '100', '110', '100', '90'],
 159             ['BM_ThirdFaster', '-0.3333', '-0.3334', '100', '67', '100', '67'],
 160             ['BM_BadTimeUnit', '-0.9000', '+0.2000', '0', '0', '0', '1'],
 161         ]
 162         json1, json2 = self.load_results()
 163         output_lines_with_header = generate_difference_report(json1, json2, use_color=False)
 164         output_lines = output_lines_with_header[2:]
 165         print("\n".join(output_lines_with_header))
 166         self.assertEqual(len(output_lines), len(expect_lines))
 167         for i in range(0, len(output_lines)):
 168             parts = [x for x in output_lines[i].split(' ') if x]
 169             self.assertEqual(len(parts), 7)
 170             self.assertEqual(parts, expect_lines[i])
 171
 172
 173 class TestReportDifferenceBetweenFamilies(unittest.TestCase):
 174     def load_result(self):
 175         import json
 176         testInputs = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'Inputs')
 177         testOutput = os.path.join(testInputs, 'test2_run.json')
 178         with open(testOutput, 'r') as f:
 179             json = json.load(f)
 180         return json
 181
 182     def test_basic(self):
 183         expect_lines = [
 184             ['.', '-0.5000', '-0.5000', '10', '5', '10', '5'],
 185             ['./4', '-0.5000', '-0.5000', '40', '20', '40', '20'],
 186             ['Prefix/.', '-0.5000', '-0.5000', '20', '10', '20', '10'],
 187             ['Prefix/./3', '-0.5000', '-0.5000', '30', '15', '30', '15'],
 188         ]
 189         json = self.load_result()
 190         json1 = filter_benchmark(json, "BM_Z.ro", ".")
 191         json2 = filter_benchmark(json, "BM_O.e", ".")
 192         output_lines_with_header = generate_difference_report(json1, json2, use_color=False)
 193         output_lines = output_lines_with_header[2:]
 194         print("\n")
 195         print("\n".join(output_lines_with_header))
 196         self.assertEqual(len(output_lines), len(expect_lines))
 197         for i in range(0, len(output_lines)):
 198             parts = [x for x in output_lines[i].split(' ') if x]
 199             self.assertEqual(len(parts), 7)
 200             self.assertEqual(parts, expect_lines[i])
 201
 202
 203 if __name__ == '__main__':
 204     unittest.main()
 205
 206 # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
 207 # kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
 208 # kate: indent-mode python; remove-trailing-spaces modified;